Merge remote-tracking branch 'upstream/master' into evmjit

accounts/abi/bind/bind.go

@@ -129,7 +129,7 @@ func Bind(types []string, abis []string, bytecodes []string, pkg string, lang La
 		return string(code), nil
 	}
 	// For all others just return as is for now
-	return string(buffer.Bytes()), nil
+	return buffer.String(), nil
 }
 
 // bindType is a set of type binders that convert Solidity types to some supported

accounts/abi/unpack_test.go

@@ -368,11 +368,11 @@ func TestUnmarshal(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	} else {
-		if bytes.Compare(p0, p0Exp) != 0 {
+		if !bytes.Equal(p0, p0Exp) {
 			t.Errorf("unexpected value unpacked: want %x, got %x", p0Exp, p0)
 		}
 
-		if bytes.Compare(p1[:], p1Exp) != 0 {
+		if !bytes.Equal(p1[:], p1Exp) {
 			t.Errorf("unexpected value unpacked: want %x, got %x", p1Exp, p1)
 		}
 	}

accounts/keystore/presale.go

@@ -58,6 +58,9 @@ func decryptPreSaleKey(fileContent []byte, password string) (key *Key, err error
 	if err != nil {
 		return nil, errors.New("invalid hex in encSeed")
 	}
+	if len(encSeedBytes) < 16 {
+		return nil, errors.New("invalid encSeed, too short")
+	}
 	iv := encSeedBytes[:16]
 	cipherText := encSeedBytes[16:]
 	/*

bmt/bmt.go

@@ -260,8 +260,7 @@ func NewTree(hasher BaseHasher, segmentSize, segmentCount int) *Tree {
 	for d := 1; d <= depth(segmentCount); d++ {
 		nodes := make([]*Node, count)
 		for i := 0; i < len(nodes); i++ {
-			var parent *Node
+			parent := prevlevel[i/2]
-			parent = prevlevel[i/2]
 			t := NewNode(level, i, parent)
 			nodes[i] = t
 		}

build/ci.go

@@ -319,8 +319,8 @@ func doLint(cmdline []string) {
 		packages = flag.CommandLine.Args()
 	}
 	// Get metalinter and install all supported linters
-	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v1"))
+	build.MustRun(goTool("get", "gopkg.in/alecthomas/gometalinter.v2"))
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), "--install")
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), "--install")
 
 	// Run fast linters batched together
 	configs := []string{
@@ -332,12 +332,12 @@ func doLint(cmdline []string) {
 		"--enable=goconst",
 		"--min-occurrences=6", // for goconst
 	}
-	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
+	build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 
 	// Run slow linters one by one
 	for _, linter := range []string{"unconvert", "gosimple"} {
 		configs = []string{"--vendor", "--deadline=10m", "--disable-all", "--enable=" + linter}
-		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v1"), append(configs, packages...)...)
+		build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)
 	}
 }
 

cmd/faucet/faucet.go

@@ -506,7 +506,7 @@ func (f *faucet) apiHandler(conn *websocket.Conn) {
 
 		// Send an error if too frequent funding, othewise a success
 		if !fund {
-			if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))); err != nil {
+			if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))); err != nil { // nolint: gosimple
 				log.Warn("Failed to send funding error to client", "err", err)
 				return
 			}

cmd/geth/consolecmd.go

@@ -120,8 +120,12 @@ func remoteConsole(ctx *cli.Context) error {
 		if ctx.GlobalIsSet(utils.DataDirFlag.Name) {
 			path = ctx.GlobalString(utils.DataDirFlag.Name)
 		}
-		if path != "" && ctx.GlobalBool(utils.TestnetFlag.Name) {
+		if path != "" {
-			path = filepath.Join(path, "testnet")
+			if ctx.GlobalBool(utils.TestnetFlag.Name) {
+				path = filepath.Join(path, "testnet")
+			} else if ctx.GlobalBool(utils.RinkebyFlag.Name) {
+				path = filepath.Join(path, "rinkeby")
+			}
 		}
 		endpoint = fmt.Sprintf("%s/geth.ipc", path)
 	}

cmd/swarm/config.go

@@ -267,7 +267,7 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 	}
 
 	//EnsApi can be set to "", so can't check for empty string, as it is allowed
-	if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists == true {
+	if ensapi, exists := os.LookupEnv(SWARM_ENV_ENS_API); exists {
 		currentConfig.EnsApi = ensapi
 	}
 

cmd/swarm/config_test.go

@@ -124,7 +124,7 @@ func TestCmdLineOverrides(t *testing.T) {
 		t.Fatalf("Expected network ID to be %d, got %d", 42, info.NetworkId)
 	}
 
-	if info.SyncEnabled != true {
+	if !info.SyncEnabled {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
@@ -219,7 +219,7 @@ func TestFileOverrides(t *testing.T) {
 		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
 	}
 
-	if info.SyncEnabled != true {
+	if !info.SyncEnabled {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
@@ -334,7 +334,7 @@ func TestEnvVars(t *testing.T) {
 		t.Fatalf("Expected Cors flag to be set to %s, got %s", "*", info.Cors)
 	}
 
-	if info.SyncEnabled != true {
+	if !info.SyncEnabled {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
@@ -431,7 +431,7 @@ func TestCmdLineOverridesFile(t *testing.T) {
 		t.Fatalf("Expected network ID to be %d, got %d", expectNetworkId, info.NetworkId)
 	}
 
-	if info.SyncEnabled != true {
+	if !info.SyncEnabled {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 

consensus/clique/clique.go

@@ -630,7 +630,7 @@ func (c *Clique) Seal(chain consensus.ChainReader, block *types.Block, stop <-ch
 		}
 	}
 	// Sweet, the protocol permits us to sign the block, wait for our time
-	delay := time.Unix(header.Time.Int64(), 0).Sub(time.Now())
+	delay := time.Unix(header.Time.Int64(), 0).Sub(time.Now()) // nolint: gosimple
 	if header.Difficulty.Cmp(diffNoTurn) == 0 {
 		// It's not our turn explicitly to sign, delay it a bit
 		wiggle := time.Duration(len(snap.Signers)/2+1) * wiggleTime

consensus/ethash/consensus.go

@@ -36,9 +36,10 @@ import (
 
 // Ethash proof-of-work protocol constants.
 var (
-	FrontierBlockReward  *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
+	FrontierBlockReward    *big.Int = big.NewInt(5e+18) // Block reward in wei for successfully mining a block
-	ByzantiumBlockReward *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
+	ByzantiumBlockReward   *big.Int = big.NewInt(3e+18) // Block reward in wei for successfully mining a block upward from Byzantium
-	maxUncles                     = 2                 // Maximum number of uncles allowed in a single block
+	maxUncles                       = 2                 // Maximum number of uncles allowed in a single block
+	allowedFutureBlockTime          = 15 * time.Second  // Max time from current time allowed for blocks, before they're considered future blocks
 )
 
 // Various error messages to mark blocks invalid. These should be private to
@@ -231,7 +232,7 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent *
 			return errLargeBlockTime
 		}
 	} else {
-		if header.Time.Cmp(big.NewInt(time.Now().Unix())) > 0 {
+		if header.Time.Cmp(big.NewInt(time.Now().Add(allowedFutureBlockTime).Unix())) > 0 {
 			return consensus.ErrFutureBlock
 		}
 	}

console/console_test.go

@@ -164,7 +164,7 @@ func TestWelcome(t *testing.T) {
 
 	tester.console.Welcome()
 
-	output := string(tester.output.Bytes())
+	output := tester.output.String()
 	if want := "Welcome"; !strings.Contains(output, want) {
 		t.Fatalf("console output missing welcome message: have\n%s\nwant also %s", output, want)
 	}
@@ -188,7 +188,7 @@ func TestEvaluate(t *testing.T) {
 	defer tester.Close(t)
 
 	tester.console.Evaluate("2 + 2")
-	if output := string(tester.output.Bytes()); !strings.Contains(output, "4") {
+	if output := tester.output.String(); !strings.Contains(output, "4") {
 		t.Fatalf("statement evaluation failed: have %s, want %s", output, "4")
 	}
 }
@@ -218,7 +218,7 @@ func TestInteractive(t *testing.T) {
 	case <-time.After(time.Second):
 		t.Fatalf("secondary prompt timeout")
 	}
-	if output := string(tester.output.Bytes()); !strings.Contains(output, "4") {
+	if output := tester.output.String(); !strings.Contains(output, "4") {
 		t.Fatalf("statement evaluation failed: have %s, want %s", output, "4")
 	}
 }
@@ -230,7 +230,7 @@ func TestPreload(t *testing.T) {
 	defer tester.Close(t)
 
 	tester.console.Evaluate("preloaded")
-	if output := string(tester.output.Bytes()); !strings.Contains(output, "some-preloaded-string") {
+	if output := tester.output.String(); !strings.Contains(output, "some-preloaded-string") {
 		t.Fatalf("preloaded variable missing: have %s, want %s", output, "some-preloaded-string")
 	}
 }
@@ -243,7 +243,7 @@ func TestExecute(t *testing.T) {
 	tester.console.Execute("exec.js")
 
 	tester.console.Evaluate("execed")
-	if output := string(tester.output.Bytes()); !strings.Contains(output, "some-executed-string") {
+	if output := tester.output.String(); !strings.Contains(output, "some-executed-string") {
 		t.Fatalf("execed variable missing: have %s, want %s", output, "some-executed-string")
 	}
 }
@@ -275,7 +275,7 @@ func TestPrettyPrint(t *testing.T) {
   string: ` + two + `
 }
 `
-	if output := string(tester.output.Bytes()); output != want {
+	if output := tester.output.String(); output != want {
 		t.Fatalf("pretty print mismatch: have %s, want %s", output, want)
 	}
 }
@@ -287,7 +287,7 @@ func TestPrettyError(t *testing.T) {
 	tester.console.Evaluate("throw 'hello'")
 
 	want := jsre.ErrorColor("hello") + "\n"
-	if output := string(tester.output.Bytes()); output != want {
+	if output := tester.output.String(); output != want {
 		t.Fatalf("pretty error mismatch: have %s, want %s", output, want)
 	}
 }

contracts/release/release.go

@@ -137,7 +137,7 @@ func (r *ReleaseService) checkVersion() {
 	if err != nil {
 		if err == bind.ErrNoCode {
 			log.Debug("Release oracle not found", "contract", r.config.Oracle)
-		} else {
+		} else if err != les.ErrNoPeers {
 			log.Error("Failed to retrieve current release", "err", err)
 		}
 		return

core/asm/asm.go

@@ -114,10 +114,7 @@ func PrintDisassembled(code string) error {
 			fmt.Printf("%06v: %v\n", it.PC(), it.Op())
 		}
 	}
-	if err := it.Error(); err != nil {
+	return it.Error()
-		return err
-	}
-	return nil
 }
 
 // Return all disassembled EVM instructions in human-readable format.

core/asm/compiler.go

@@ -237,10 +237,7 @@ func (c *Compiler) pushBin(v interface{}) {
 // isPush returns whether the string op is either any of
 // push(N).
 func isPush(op string) bool {
-	if op == "push" {
+	return op == "push"
-		return true
-	}
-	return false
 }
 
 // isJump returns whether the string op is jump(i)

core/vm/evm.go

@@ -38,7 +38,7 @@ type (
 )
 
 // run runs the given contract and takes care of running precompiles with a fallback to the byte code interpreter.
-func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, error) {
+func run(evm *EVM, contract *Contract, input []byte) ([]byte, error) {
 	if contract.CodeAddr != nil {
 		precompiles := PrecompiledContractsHomestead
 		if evm.ChainConfig().IsByzantium(evm.BlockNumber) {
@@ -48,7 +48,7 @@ func run(evm *EVM, snapshot int, contract *Contract, input []byte) ([]byte, erro
 			return RunPrecompiledContract(p, input, contract)
 		}
 	}
-	return evm.interpreter.Run(snapshot, contract, input)
+	return evm.interpreter.Run(contract, input)
 }
 
 // Context provides the EVM with auxiliary information. Once provided
@@ -171,7 +171,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in homestead this also counts for code storage gas errors.
@@ -215,7 +215,7 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 	contract := NewContract(caller, to, value, gas)
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -248,7 +248,7 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by
 	contract := NewContract(caller, to, nil, gas).AsDelegate()
 	contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr))
 
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -291,7 +291,7 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 	// When an error was returned by the EVM or when setting the creation code
 	// above we revert to the snapshot and consume any gas remaining. Additionally
 	// when we're in Homestead this also counts for code storage gas errors.
-	ret, err = run(evm, snapshot, contract, input)
+	ret, err = run(evm, contract, input)
 	if err != nil {
 		evm.StateDB.RevertToSnapshot(snapshot)
 		if err != errExecutionReverted {
@@ -338,7 +338,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I
 	if evm.vmConfig.NoRecursion && evm.depth > 0 {
 		return nil, contractAddr, gas, nil
 	}
-	ret, err = run(evm, snapshot, contract, nil)
+	ret, err = run(evm, contract, nil)
 	// check whether the max code size has been exceeded
 	maxCodeSizeExceeded := evm.ChainConfig().IsEIP158(evm.BlockNumber) && len(ret) > params.MaxCodeSize
 	// if the contract creation ran successfully and no errors were returned

core/vm/interpreter.go

@@ -107,9 +107,9 @@ func (in *Interpreter) enforceRestrictions(op OpCode, operation operation, stack
 // the return byte-slice and an error if one occurred.
 //
 // It's important to note that any errors returned by the interpreter should be
-// considered a revert-and-consume-all-gas operation. No error specific checks
+// considered a revert-and-consume-all-gas operation except for
-// should be handled to reduce complexity and errors further down the in.
+// errExecutionReverted which means revert-and-keep-gas-left.
-func (in *Interpreter) Run(snapshot int, contract *Contract, input []byte) (ret []byte, err error) {
+func (in *Interpreter) Run(contract *Contract, input []byte) (ret []byte, err error) {
 	// Increment the call depth which is restricted to 1024
 	in.evm.depth++
 	defer func() { in.evm.depth-- }()

crypto/crypto.go

@@ -79,7 +79,7 @@ func ToECDSA(d []byte) (*ecdsa.PrivateKey, error) {
 	return toECDSA(d, true)
 }
 
-// ToECDSAUnsafe blidly converts a binary blob to a private key. It should almost
+// ToECDSAUnsafe blindly converts a binary blob to a private key. It should almost
 // never be used unless you are sure the input is valid and want to avoid hitting
 // errors due to bad origin encoding (0 prefixes cut off).
 func ToECDSAUnsafe(d []byte) *ecdsa.PrivateKey {

crypto/secp256k1/curve.go

@@ -34,7 +34,6 @@ package secp256k1
 import (
 	"crypto/elliptic"
 	"math/big"
-	"sync"
 	"unsafe"
 
 	"github.com/ethereum/go-ethereum/common/math"
@@ -42,7 +41,7 @@ import (
 
 /*
 #include "libsecp256k1/include/secp256k1.h"
-extern int secp256k1_pubkey_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar);
+extern int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, const unsigned char *point, const unsigned char *scalar);
 */
 import "C"
 
@@ -236,7 +235,7 @@ func (BitCurve *BitCurve) ScalarMult(Bx, By *big.Int, scalar []byte) (*big.Int,
 	math.ReadBits(By, point[32:])
 	pointPtr := (*C.uchar)(unsafe.Pointer(&point[0]))
 	scalarPtr := (*C.uchar)(unsafe.Pointer(&scalar[0]))
-	res := C.secp256k1_pubkey_scalar_mul(context, pointPtr, scalarPtr)
+	res := C.secp256k1_ext_scalar_mul(context, pointPtr, scalarPtr)
 
 	// Unpack the result and clear temporaries.
 	x := new(big.Int).SetBytes(point[:32])
@@ -263,14 +262,10 @@ func (BitCurve *BitCurve) ScalarBaseMult(k []byte) (*big.Int, *big.Int) {
 // X9.62.
 func (BitCurve *BitCurve) Marshal(x, y *big.Int) []byte {
 	byteLen := (BitCurve.BitSize + 7) >> 3
-
 	ret := make([]byte, 1+2*byteLen)
-	ret[0] = 4 // uncompressed point
+	ret[0] = 4 // uncompressed point flag
-
+	math.ReadBits(x, ret[1:1+byteLen])
-	xBytes := x.Bytes()
+	math.ReadBits(y, ret[1+byteLen:])
-	copy(ret[1+byteLen-len(xBytes):], xBytes)
-	yBytes := y.Bytes()
-	copy(ret[1+2*byteLen-len(yBytes):], yBytes)
 	return ret
 }
 
@@ -289,24 +284,21 @@ func (BitCurve *BitCurve) Unmarshal(data []byte) (x, y *big.Int) {
 	return
 }
 
-var (
+var theCurve = new(BitCurve)
-	initonce sync.Once
-	theCurve *BitCurve
-)
 
-// S256 returns a BitCurve which implements secp256k1 (see SEC 2 section 2.7.1)
+func init() {
+	// See SEC 2 section 2.7.1
+	// curve parameters taken from:
+	// http://www.secg.org/collateral/sec2_final.pdf
+	theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
+	theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
+	theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
+	theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
+	theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
+	theCurve.BitSize = 256
+}
+
+// S256 returns a BitCurve which implements secp256k1.
 func S256() *BitCurve {
-	initonce.Do(func() {
-		// See SEC 2 section 2.7.1
-		// curve parameters taken from:
-		// http://www.secg.org/collateral/sec2_final.pdf
-		theCurve = new(BitCurve)
-		theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
-		theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
-		theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
-		theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
-		theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
-		theCurve.BitSize = 256
-	})
 	return theCurve
 }

crypto/secp256k1/ext.h

@@ -19,7 +19,7 @@ static secp256k1_context* secp256k1_context_create_sign_verify() {
 	return secp256k1_context_create(SECP256K1_CONTEXT_SIGN | SECP256K1_CONTEXT_VERIFY);
 }
 
-// secp256k1_ecdsa_recover_pubkey recovers the public key of an encoded compact signature.
+// secp256k1_ext_ecdsa_recover recovers the public key of an encoded compact signature.
 //
 // Returns: 1: recovery was successful
 //          0: recovery was not successful
@@ -27,7 +27,7 @@ static secp256k1_context* secp256k1_context_create_sign_verify() {
 //  Out:    pubkey_out: the serialized 65-byte public key of the signer (cannot be NULL)
 //  In:     sigdata:    pointer to a 65-byte signature with the recovery id at the end (cannot be NULL)
 //          msgdata:    pointer to a 32-byte message (cannot be NULL)
-static int secp256k1_ecdsa_recover_pubkey(
+static int secp256k1_ext_ecdsa_recover(
 	const secp256k1_context* ctx,
 	unsigned char *pubkey_out,
 	const unsigned char *sigdata,
@@ -46,7 +46,7 @@ static int secp256k1_ecdsa_recover_pubkey(
 	return secp256k1_ec_pubkey_serialize(ctx, pubkey_out, &outputlen, &pubkey, SECP256K1_EC_UNCOMPRESSED);
 }
 
-// secp256k1_ecdsa_verify_enc verifies an encoded compact signature.
+// secp256k1_ext_ecdsa_verify verifies an encoded compact signature.
 //
 // Returns: 1: signature is valid
 //          0: signature is invalid
@@ -55,7 +55,7 @@ static int secp256k1_ecdsa_recover_pubkey(
 //          msgdata:    pointer to a 32-byte message (cannot be NULL)
 //          pubkeydata: pointer to public key data (cannot be NULL)
 //          pubkeylen:  length of pubkeydata
-static int secp256k1_ecdsa_verify_enc(
+static int secp256k1_ext_ecdsa_verify(
 	const secp256k1_context* ctx,
 	const unsigned char *sigdata,
 	const unsigned char *msgdata,
@@ -74,28 +74,34 @@ static int secp256k1_ecdsa_verify_enc(
 	return secp256k1_ecdsa_verify(ctx, &sig, msgdata, &pubkey);
 }
 
-// secp256k1_decompress_pubkey decompresses a public key.
+// secp256k1_ext_reencode_pubkey decodes then encodes a public key. It can be used to
+// convert between public key formats. The input/output formats are chosen depending on the
+// length of the input/output buffers.
 //
-// Returns: 1: public key is valid
+// Returns: 1: conversion successful
-//          0: public key is invalid
+//          0: conversion unsuccessful
 // Args:    ctx:        pointer to a context object (cannot be NULL)
-//  Out:    pubkey_out: the serialized 65-byte public key (cannot be NULL)
+//  Out:    out:        output buffer that will contain the reencoded key (cannot be NULL)
-//  In:     pubkeydata: pointer to 33 bytes of compressed public key data (cannot be NULL)
+//  In:     outlen:     length of out (33 for compressed keys, 65 for uncompressed keys)
-static int secp256k1_decompress_pubkey(
+//          pubkeydata: the input public key (cannot be NULL)
+//          pubkeylen:  length of pubkeydata
+static int secp256k1_ext_reencode_pubkey(
 	const secp256k1_context* ctx,
-	unsigned char *pubkey_out,
+	unsigned char *out,
-	const unsigned char *pubkeydata
+	size_t outlen,
+	const unsigned char *pubkeydata,
+	size_t pubkeylen
 ) {
 	secp256k1_pubkey pubkey;
 
-	if (!secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeydata, 33)) {
+	if (!secp256k1_ec_pubkey_parse(ctx, &pubkey, pubkeydata, pubkeylen)) {
 		return 0;
 	}
-	size_t outputlen = 65;
+	unsigned int flag = (outlen == 33) ? SECP256K1_EC_COMPRESSED : SECP256K1_EC_UNCOMPRESSED;
-	return secp256k1_ec_pubkey_serialize(ctx, pubkey_out, &outputlen, &pubkey, SECP256K1_EC_UNCOMPRESSED);
+	return secp256k1_ec_pubkey_serialize(ctx, out, &outlen, &pubkey, flag);
 }
 
-// secp256k1_pubkey_scalar_mul multiplies a point by a scalar in constant time.
+// secp256k1_ext_scalar_mul multiplies a point by a scalar in constant time.
 //
 // Returns: 1: multiplication was successful
 //          0: scalar was invalid (zero or overflow)
@@ -104,7 +110,7 @@ static int secp256k1_decompress_pubkey(
 //  In:     point:    pointer to a 64-byte public point,
 //                    encoded as two 256bit big-endian numbers.
 //          scalar:   a 32-byte scalar with which to multiply the point
-int secp256k1_pubkey_scalar_mul(const secp256k1_context* ctx, unsigned char *point, const unsigned char *scalar) {
+int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, unsigned char *point, const unsigned char *scalar) {
 	int ret = 0;
 	int overflow = 0;
 	secp256k1_fe feX, feY;

crypto/secp256k1/secp256.go

@@ -115,7 +115,7 @@ func RecoverPubkey(msg []byte, sig []byte) ([]byte, error) {
 		sigdata = (*C.uchar)(unsafe.Pointer(&sig[0]))
 		msgdata = (*C.uchar)(unsafe.Pointer(&msg[0]))
 	)
-	if C.secp256k1_ecdsa_recover_pubkey(context, (*C.uchar)(unsafe.Pointer(&pubkey[0])), sigdata, msgdata) == 0 {
+	if C.secp256k1_ext_ecdsa_recover(context, (*C.uchar)(unsafe.Pointer(&pubkey[0])), sigdata, msgdata) == 0 {
 		return nil, ErrRecoverFailed
 	}
 	return pubkey, nil
@@ -130,22 +130,42 @@ func VerifySignature(pubkey, msg, signature []byte) bool {
 	sigdata := (*C.uchar)(unsafe.Pointer(&signature[0]))
 	msgdata := (*C.uchar)(unsafe.Pointer(&msg[0]))
 	keydata := (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-	return C.secp256k1_ecdsa_verify_enc(context, sigdata, msgdata, keydata, C.size_t(len(pubkey))) != 0
+	return C.secp256k1_ext_ecdsa_verify(context, sigdata, msgdata, keydata, C.size_t(len(pubkey))) != 0
 }
 
 // DecompressPubkey parses a public key in the 33-byte compressed format.
 // It returns non-nil coordinates if the public key is valid.
-func DecompressPubkey(pubkey []byte) (X, Y *big.Int) {
+func DecompressPubkey(pubkey []byte) (x, y *big.Int) {
 	if len(pubkey) != 33 {
 		return nil, nil
 	}
-	buf := make([]byte, 65)
+	var (
-	bufdata := (*C.uchar)(unsafe.Pointer(&buf[0]))
+		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
-	pubkeydata := (*C.uchar)(unsafe.Pointer(&pubkey[0]))
+		pubkeylen  = C.size_t(len(pubkey))
-	if C.secp256k1_decompress_pubkey(context, bufdata, pubkeydata) == 0 {
+		out        = make([]byte, 65)
+		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
+		outlen     = C.size_t(len(out))
+	)
+	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
 		return nil, nil
 	}
-	return new(big.Int).SetBytes(buf[1:33]), new(big.Int).SetBytes(buf[33:])
+	return new(big.Int).SetBytes(out[1:33]), new(big.Int).SetBytes(out[33:])
+}
+
+// CompressPubkey encodes a public key to 33-byte compressed format.
+func CompressPubkey(x, y *big.Int) []byte {
+	var (
+		pubkey     = S256().Marshal(x, y)
+		pubkeydata = (*C.uchar)(unsafe.Pointer(&pubkey[0]))
+		pubkeylen  = C.size_t(len(pubkey))
+		out        = make([]byte, 33)
+		outdata    = (*C.uchar)(unsafe.Pointer(&out[0]))
+		outlen     = C.size_t(len(out))
+	)
+	if C.secp256k1_ext_reencode_pubkey(context, outdata, outlen, pubkeydata, pubkeylen) == 0 {
+		panic("libsecp256k1 error")
+	}
+	return out
 }
 
 func checkSignature(sig []byte) error {

crypto/signature_cgo.go

@@ -76,6 +76,11 @@ func DecompressPubkey(pubkey []byte) (*ecdsa.PublicKey, error) {
 	return &ecdsa.PublicKey{X: x, Y: y, Curve: S256()}, nil
 }
 
+// CompressPubkey encodes a public key to the 33-byte compressed format.
+func CompressPubkey(pubkey *ecdsa.PublicKey) []byte {
+	return secp256k1.CompressPubkey(pubkey.X, pubkey.Y)
+}
+
 // S256 returns an instance of the secp256k1 curve.
 func S256() elliptic.Curve {
 	return secp256k1.S256()

crypto/signature_nocgo.go

@@ -102,6 +102,11 @@ func DecompressPubkey(pubkey []byte) (*ecdsa.PublicKey, error) {
 	return key.ToECDSA(), nil
 }
 
+// CompressPubkey encodes a public key to the 33-byte compressed format.
+func CompressPubkey(pubkey *ecdsa.PublicKey) []byte {
+	return (*btcec.PublicKey)(pubkey).SerializeCompressed()
+}
+
 // S256 returns an instance of the secp256k1 curve.
 func S256() elliptic.Curve {
 	return btcec.S256()

crypto/signature_test.go

@@ -18,10 +18,13 @@ package crypto
 
 import (
 	"bytes"
+	"crypto/ecdsa"
+	"reflect"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/common/math"
 )
 
 var (
@@ -65,6 +68,11 @@ func TestVerifySignature(t *testing.T) {
 	if VerifySignature(testpubkey, testmsg, sig[:len(sig)-2]) {
 		t.Errorf("signature valid even though it's incomplete")
 	}
+	wrongkey := common.CopyBytes(testpubkey)
+	wrongkey[10]++
+	if VerifySignature(wrongkey, testmsg, sig) {
+		t.Errorf("signature valid with with wrong public key")
+	}
 }
 
 func TestDecompressPubkey(t *testing.T) {
@@ -86,6 +94,36 @@ func TestDecompressPubkey(t *testing.T) {
 	}
 }
 
+func TestCompressPubkey(t *testing.T) {
+	key := &ecdsa.PublicKey{
+		Curve: S256(),
+		X:     math.MustParseBig256("0xe32df42865e97135acfb65f3bae71bdc86f4d49150ad6a440b6f15878109880a"),
+		Y:     math.MustParseBig256("0x0a2b2667f7e725ceea70c673093bf67663e0312623c8e091b13cf2c0f11ef652"),
+	}
+	compressed := CompressPubkey(key)
+	if !bytes.Equal(compressed, testpubkeyc) {
+		t.Errorf("wrong public key result: got %x, want %x", compressed, testpubkeyc)
+	}
+}
+
+func TestPubkeyRandom(t *testing.T) {
+	const runs = 200
+
+	for i := 0; i < runs; i++ {
+		key, err := GenerateKey()
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		pubkey2, err := DecompressPubkey(CompressPubkey(&key.PublicKey))
+		if err != nil {
+			t.Fatalf("iteration %d: %v", i, err)
+		}
+		if !reflect.DeepEqual(key.PublicKey, *pubkey2) {
+			t.Fatalf("iteration %d: keys not equal", i)
+		}
+	}
+}
+
 func BenchmarkEcrecoverSignature(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		if _, err := Ecrecover(testmsg, testsig); err != nil {

ethstats/ethstats.go

@@ -193,7 +193,6 @@ func (s *Service) loop() {
 			}
 		}
 		close(quitCh)
-		return
 	}()
 	// Loop reporting until termination
 	for {

internal/ethapi/api.go

@@ -17,6 +17,7 @@
 package ethapi
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -1003,9 +1004,12 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 func (s *PublicTransactionPoolAPI) GetTransactionReceipt(hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
-		return nil, nil
+		return nil, errors.New("unknown transaction")
 	}
 	receipt, _, _, _ := core.GetReceipt(s.b.ChainDb(), hash) // Old receipts don't have the lookup data available
+	if receipt == nil {
+		return nil, errors.New("unknown receipt")
+	}
 
 	var signer types.Signer = types.FrontierSigner{}
 	if tx.Protected() {
@@ -1067,11 +1071,14 @@ type SendTxArgs struct {
 	Gas      *hexutil.Big    `json:"gas"`
 	GasPrice *hexutil.Big    `json:"gasPrice"`
 	Value    *hexutil.Big    `json:"value"`
-	Data     hexutil.Bytes   `json:"data"`
 	Nonce    *hexutil.Uint64 `json:"nonce"`
+	// We accept "data" and "input" for backwards-compatibility reasons. "input" is the
+	// newer name and should be preferred by clients.
+	Data  *hexutil.Bytes `json:"data"`
+	Input *hexutil.Bytes `json:"input"`
 }
 
-// prepareSendTxArgs is a helper function that fills in default values for unspecified tx fields.
+// setDefaults is a helper function that fills in default values for unspecified tx fields.
 func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error {
 	if args.Gas == nil {
 		args.Gas = (*hexutil.Big)(big.NewInt(defaultGas))
@@ -1093,14 +1100,23 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error {
 		}
 		args.Nonce = (*hexutil.Uint64)(&nonce)
 	}
+	if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) {
+		return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`)
+	}
 	return nil
 }
 
 func (args *SendTxArgs) toTransaction() *types.Transaction {
-	if args.To == nil {
+	var input []byte
-		return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), args.Data)
+	if args.Data != nil {
+		input = *args.Data
+	} else if args.Input != nil {
+		input = *args.Input
 	}
-	return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), args.Data)
+	if args.To == nil {
+		return types.NewContractCreation(uint64(*args.Nonce), (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), input)
+	}
+	return types.NewTransaction(uint64(*args.Nonce), *args.To, (*big.Int)(args.Value), (*big.Int)(args.Gas), (*big.Int)(args.GasPrice), input)
 }
 
 // submitTransaction is a helper function that submits tx to txPool and logs a message.

internal/ethapi/tracer_test.go

@@ -48,7 +48,7 @@ func runTrace(tracer *JavascriptTracer) (interface{}, error) {
 	contract := vm.NewContract(account{}, account{}, big.NewInt(0), 10000)
 	contract.Code = []byte{byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x1, 0x0}
 
-	_, err := env.Interpreter().Run(0, contract, []byte{})
+	_, err := env.Interpreter().Run(contract, []byte{})
 	if err != nil {
 		return nil, err
 	}

light/trie.go

@@ -151,7 +151,7 @@ func (t *odrTrie) do(key []byte, fn func() error) error {
 		}
 		r := &TrieRequest{Id: t.id, Key: key}
 		if err := t.db.backend.Retrieve(t.db.ctx, r); err != nil {
-			return fmt.Errorf("can't fetch trie key %x: %v", key, err)
+			return err
 		}
 	}
 }

p2p/discover/database.go

@@ -226,14 +226,14 @@ func (db *nodeDB) ensureExpirer() {
 // expirer should be started in a go routine, and is responsible for looping ad
 // infinitum and dropping stale data from the database.
 func (db *nodeDB) expirer() {
-	tick := time.Tick(nodeDBCleanupCycle)
+	tick := time.NewTicker(nodeDBCleanupCycle)
+	defer tick.Stop()
 	for {
 		select {
-		case <-tick:
+		case <-tick.C:
 			if err := db.expireNodes(); err != nil {
 				log.Error("Failed to expire nodedb items", "err", err)
 			}
-
 		case <-db.quit:
 			return
 		}

p2p/discv5/net.go

@@ -684,7 +684,7 @@ func (net *Network) refresh(done chan<- struct{}) {
 		seeds = net.nursery
 	}
 	if len(seeds) == 0 {
-		log.Trace(fmt.Sprint("no seed nodes found"))
+		log.Trace("no seed nodes found")
 		close(done)
 		return
 	}

p2p/discv5/ntp.go

@@ -54,10 +54,10 @@ func checkClockDrift() {
 		howtofix := fmt.Sprintf("Please enable network time synchronisation in system settings")
 		separator := strings.Repeat("-", len(warning))
 
-		log.Warn(fmt.Sprint(separator))
+		log.Warn(separator)
-		log.Warn(fmt.Sprint(warning))
+		log.Warn(warning)
-		log.Warn(fmt.Sprint(howtofix))
+		log.Warn(howtofix)
-		log.Warn(fmt.Sprint(separator))
+		log.Warn(separator)
 	} else {
 		log.Debug(fmt.Sprintf("Sanity NTP check reported %v drift, all ok", drift))
 	}

p2p/discv5/ticket.go

@@ -398,12 +398,12 @@ func (s *ticketStore) nextRegisterableTicket() (t *ticketRef, wait time.Duration
 			//s.removeExcessTickets(topic)
 			if len(tickets.buckets) != 0 {
 				empty = false
-				if list := tickets.buckets[bucket]; list != nil {
+
-					for _, ref := range list {
+				list := tickets.buckets[bucket]
-						//debugLog(fmt.Sprintf(" nrt bucket = %d node = %x sn = %v wait = %v", bucket, ref.t.node.ID[:8], ref.t.serial, time.Duration(ref.topicRegTime()-now)))
+				for _, ref := range list {
-						if nextTicket.t == nil || ref.topicRegTime() < nextTicket.topicRegTime() {
+					//debugLog(fmt.Sprintf(" nrt bucket = %d node = %x sn = %v wait = %v", bucket, ref.t.node.ID[:8], ref.t.serial, time.Duration(ref.topicRegTime()-now)))
-							nextTicket = ref
+					if nextTicket.t == nil || ref.topicRegTime() < nextTicket.topicRegTime() {
-						}
+						nextTicket = ref
 					}
 				}
 			}

p2p/simulations/adapters/state.go

@@ -0,0 +1,35 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package adapters
+
+type SimStateStore struct {
+	m map[string][]byte
+}
+
+func (self *SimStateStore) Load(s string) ([]byte, error) {
+	return self.m[s], nil
+}
+
+func (self *SimStateStore) Save(s string, data []byte) error {
+	self.m[s] = data
+	return nil
+}
+
+func NewSimStateStore() *SimStateStore {
+	return &SimStateStore{
+		make(map[string][]byte),
+	}
+}

p2p/simulations/http.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/p2p"
@@ -263,8 +264,10 @@ func (c *Client) Send(method, path string, in, out interface{}) error {
 
 // Server is an HTTP server providing an API to manage a simulation network
 type Server struct {
-	router  *httprouter.Router
+	router     *httprouter.Router
-	network *Network
+	network    *Network
+	mockerStop chan struct{} // when set, stops the current mocker
+	mockerMtx  sync.Mutex    // synchronises access to the mockerStop field
 }
 
 // NewServer returns a new simulation API server
@@ -278,6 +281,10 @@ func NewServer(network *Network) *Server {
 	s.GET("/", s.GetNetwork)
 	s.POST("/start", s.StartNetwork)
 	s.POST("/stop", s.StopNetwork)
+	s.POST("/mocker/start", s.StartMocker)
+	s.POST("/mocker/stop", s.StopMocker)
+	s.GET("/mocker", s.GetMockers)
+	s.POST("/reset", s.ResetNetwork)
 	s.GET("/events", s.StreamNetworkEvents)
 	s.GET("/snapshot", s.CreateSnapshot)
 	s.POST("/snapshot", s.LoadSnapshot)
@@ -318,6 +325,59 @@ func (s *Server) StopNetwork(w http.ResponseWriter, req *http.Request) {
 	w.WriteHeader(http.StatusOK)
 }
 
+// StartMocker starts the mocker node simulation
+func (s *Server) StartMocker(w http.ResponseWriter, req *http.Request) {
+	s.mockerMtx.Lock()
+	defer s.mockerMtx.Unlock()
+	if s.mockerStop != nil {
+		http.Error(w, "mocker already running", http.StatusInternalServerError)
+		return
+	}
+	mockerType := req.FormValue("mocker-type")
+	mockerFn := LookupMocker(mockerType)
+	if mockerFn == nil {
+		http.Error(w, fmt.Sprintf("unknown mocker type %q", mockerType), http.StatusBadRequest)
+		return
+	}
+	nodeCount, err := strconv.Atoi(req.FormValue("node-count"))
+	if err != nil {
+		http.Error(w, "invalid node-count provided", http.StatusBadRequest)
+		return
+	}
+	s.mockerStop = make(chan struct{})
+	go mockerFn(s.network, s.mockerStop, nodeCount)
+
+	w.WriteHeader(http.StatusOK)
+}
+
+// StopMocker stops the mocker node simulation
+func (s *Server) StopMocker(w http.ResponseWriter, req *http.Request) {
+	s.mockerMtx.Lock()
+	defer s.mockerMtx.Unlock()
+	if s.mockerStop == nil {
+		http.Error(w, "stop channel not initialized", http.StatusInternalServerError)
+		return
+	}
+	close(s.mockerStop)
+	s.mockerStop = nil
+
+	w.WriteHeader(http.StatusOK)
+}
+
+// GetMockerList returns a list of available mockers
+func (s *Server) GetMockers(w http.ResponseWriter, req *http.Request) {
+
+	list := GetMockerList()
+	s.JSON(w, http.StatusOK, list)
+}
+
+// ResetNetwork resets all properties of a network to its initial (empty) state
+func (s *Server) ResetNetwork(w http.ResponseWriter, req *http.Request) {
+	s.network.Reset()
+
+	w.WriteHeader(http.StatusOK)
+}
+
 // StreamNetworkEvents streams network events as a server-sent-events stream
 func (s *Server) StreamNetworkEvents(w http.ResponseWriter, req *http.Request) {
 	events := make(chan *Event)

p2p/simulations/mocker.go

@@ -0,0 +1,192 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package simulations simulates p2p networks.
+// A mocker simulates starting and stopping real nodes in a network.
+package simulations
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+//a map of mocker names to its function
+var mockerList = map[string]func(net *Network, quit chan struct{}, nodeCount int){
+	"startStop":     startStop,
+	"probabilistic": probabilistic,
+	"boot":          boot,
+}
+
+//Lookup a mocker by its name, returns the mockerFn
+func LookupMocker(mockerType string) func(net *Network, quit chan struct{}, nodeCount int) {
+	return mockerList[mockerType]
+}
+
+//Get a list of mockers (keys of the map)
+//Useful for frontend to build available mocker selection
+func GetMockerList() []string {
+	list := make([]string, 0, len(mockerList))
+	for k := range mockerList {
+		list = append(list, k)
+	}
+	return list
+}
+
+//The boot mockerFn only connects the node in a ring and doesn't do anything else
+func boot(net *Network, quit chan struct{}, nodeCount int) {
+	_, err := connectNodesInRing(net, nodeCount)
+	if err != nil {
+		panic("Could not startup node network for mocker")
+	}
+}
+
+//The startStop mockerFn stops and starts nodes in a defined period (ticker)
+func startStop(net *Network, quit chan struct{}, nodeCount int) {
+	nodes, err := connectNodesInRing(net, nodeCount)
+	if err != nil {
+		panic("Could not startup node network for mocker")
+	}
+	tick := time.NewTicker(10 * time.Second)
+	defer tick.Stop()
+	for {
+		select {
+		case <-quit:
+			log.Info("Terminating simulation loop")
+			return
+		case <-tick.C:
+			id := nodes[rand.Intn(len(nodes))]
+			log.Info("stopping node", "id", id)
+			if err := net.Stop(id); err != nil {
+				log.Error("error stopping node", "id", id, "err", err)
+				return
+			}
+
+			select {
+			case <-quit:
+				log.Info("Terminating simulation loop")
+				return
+			case <-time.After(3 * time.Second):
+			}
+
+			log.Debug("starting node", "id", id)
+			if err := net.Start(id); err != nil {
+				log.Error("error starting node", "id", id, "err", err)
+				return
+			}
+		}
+	}
+}
+
+//The probabilistic mocker func has a more probabilistic pattern
+//(the implementation could probably be improved):
+//nodes are connected in a ring, then a varying number of random nodes is selected,
+//mocker then stops and starts them in random intervals, and continues the loop
+func probabilistic(net *Network, quit chan struct{}, nodeCount int) {
+	nodes, err := connectNodesInRing(net, nodeCount)
+	if err != nil {
+		panic("Could not startup node network for mocker")
+	}
+	for {
+		select {
+		case <-quit:
+			log.Info("Terminating simulation loop")
+			return
+		default:
+		}
+		var lowid, highid int
+		var wg sync.WaitGroup
+		randWait := time.Duration(rand.Intn(5000)+1000) * time.Millisecond
+		rand1 := rand.Intn(nodeCount - 1)
+		rand2 := rand.Intn(nodeCount - 1)
+		if rand1 < rand2 {
+			lowid = rand1
+			highid = rand2
+		} else if rand1 > rand2 {
+			highid = rand1
+			lowid = rand2
+		} else {
+			if rand1 == 0 {
+				rand2 = 9
+			} else if rand1 == 9 {
+				rand1 = 0
+			}
+			lowid = rand1
+			highid = rand2
+		}
+		var steps = highid - lowid
+		wg.Add(steps)
+		for i := lowid; i < highid; i++ {
+			select {
+			case <-quit:
+				log.Info("Terminating simulation loop")
+				return
+			case <-time.After(randWait):
+			}
+			log.Debug(fmt.Sprintf("node %v shutting down", nodes[i]))
+			err := net.Stop(nodes[i])
+			if err != nil {
+				log.Error(fmt.Sprintf("Error stopping node %s", nodes[i]))
+				wg.Done()
+				continue
+			}
+			go func(id discover.NodeID) {
+				time.Sleep(randWait)
+				err := net.Start(id)
+				if err != nil {
+					log.Error(fmt.Sprintf("Error starting node %s", id))
+				}
+				wg.Done()
+			}(nodes[i])
+		}
+		wg.Wait()
+	}
+
+}
+
+//connect nodeCount number of nodes in a ring
+func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error) {
+	ids := make([]discover.NodeID, nodeCount)
+	for i := 0; i < nodeCount; i++ {
+		node, err := net.NewNode()
+		if err != nil {
+			log.Error("Error creating a node! %s", err)
+			return nil, err
+		}
+		ids[i] = node.ID()
+	}
+
+	for _, id := range ids {
+		if err := net.Start(id); err != nil {
+			log.Error("Error starting a node! %s", err)
+			return nil, err
+		}
+		log.Debug(fmt.Sprintf("node %v starting up", id))
+	}
+	for i, id := range ids {
+		peerID := ids[(i+1)%len(ids)]
+		if err := net.Connect(id, peerID); err != nil {
+			log.Error("Error connecting a node to a peer! %s", err)
+			return nil, err
+		}
+	}
+
+	return ids, nil
+}

p2p/simulations/mocker_test.go

@@ -0,0 +1,171 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package simulations simulates p2p networks.
+// A mokcer simulates starting and stopping real nodes in a network.
+package simulations
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+func TestMocker(t *testing.T) {
+	//start the simulation HTTP server
+	_, s := testHTTPServer(t)
+	defer s.Close()
+
+	//create a client
+	client := NewClient(s.URL)
+
+	//start the network
+	err := client.StartNetwork()
+	if err != nil {
+		t.Fatalf("Could not start test network: %s", err)
+	}
+	//stop the network to terminate
+	defer func() {
+		err = client.StopNetwork()
+		if err != nil {
+			t.Fatalf("Could not stop test network: %s", err)
+		}
+	}()
+
+	//get the list of available mocker types
+	resp, err := http.Get(s.URL + "/mocker")
+	if err != nil {
+		t.Fatalf("Could not get mocker list: %s", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 {
+		t.Fatalf("Invalid Status Code received, expected 200, got %d", resp.StatusCode)
+	}
+
+	//check the list is at least 1 in size
+	var mockerlist []string
+	err = json.NewDecoder(resp.Body).Decode(&mockerlist)
+	if err != nil {
+		t.Fatalf("Error decoding JSON mockerlist: %s", err)
+	}
+
+	if len(mockerlist) < 1 {
+		t.Fatalf("No mockers available")
+	}
+
+	nodeCount := 10
+	var wg sync.WaitGroup
+
+	events := make(chan *Event, 10)
+	var opts SubscribeOpts
+	sub, err := client.SubscribeNetwork(events, opts)
+	defer sub.Unsubscribe()
+	//wait until all nodes are started and connected
+	//store every node up event in a map (value is irrelevant, mimic Set datatype)
+	nodemap := make(map[discover.NodeID]bool)
+	wg.Add(1)
+	nodesComplete := false
+	connCount := 0
+	go func() {
+		for {
+			select {
+			case event := <-events:
+				//if the event is a node Up event only
+				if event.Node != nil && event.Node.Up {
+					//add the correspondent node ID to the map
+					nodemap[event.Node.Config.ID] = true
+					//this means all nodes got a nodeUp event, so we can continue the test
+					if len(nodemap) == nodeCount {
+						nodesComplete = true
+						//wait for 3s as the mocker will need time to connect the nodes
+						//time.Sleep( 3 *time.Second)
+					}
+				} else if event.Conn != nil && nodesComplete {
+					connCount += 1
+					if connCount == (nodeCount-1)*2 {
+						wg.Done()
+						return
+					}
+				}
+			case <-time.After(30 * time.Second):
+				wg.Done()
+				t.Fatalf("Timeout waiting for nodes being started up!")
+			}
+		}
+	}()
+
+	//take the last element of the mockerlist as the default mocker-type to ensure one is enabled
+	mockertype := mockerlist[len(mockerlist)-1]
+	//still, use hardcoded "probabilistic" one if available ;)
+	for _, m := range mockerlist {
+		if m == "probabilistic" {
+			mockertype = m
+			break
+		}
+	}
+	//start the mocker with nodeCount number of nodes
+	resp, err = http.PostForm(s.URL+"/mocker/start", url.Values{"mocker-type": {mockertype}, "node-count": {strconv.Itoa(nodeCount)}})
+	if err != nil {
+		t.Fatalf("Could not start mocker: %s", err)
+	}
+	if resp.StatusCode != 200 {
+		t.Fatalf("Invalid Status Code received for starting mocker, expected 200, got %d", resp.StatusCode)
+	}
+
+	wg.Wait()
+
+	//check there are nodeCount number of nodes in the network
+	nodes_info, err := client.GetNodes()
+	if err != nil {
+		t.Fatalf("Could not get nodes list: %s", err)
+	}
+
+	if len(nodes_info) != nodeCount {
+		t.Fatalf("Expected %d number of nodes, got: %d", nodeCount, len(nodes_info))
+	}
+
+	//stop the mocker
+	resp, err = http.Post(s.URL+"/mocker/stop", "", nil)
+	if err != nil {
+		t.Fatalf("Could not stop mocker: %s", err)
+	}
+	if resp.StatusCode != 200 {
+		t.Fatalf("Invalid Status Code received for stopping mocker, expected 200, got %d", resp.StatusCode)
+	}
+
+	//reset the network
+	_, err = http.Post(s.URL+"/reset", "", nil)
+	if err != nil {
+		t.Fatalf("Could not reset network: %s", err)
+	}
+
+	//now the number of nodes in the network should be zero
+	nodes_info, err = client.GetNodes()
+	if err != nil {
+		t.Fatalf("Could not get nodes list: %s", err)
+	}
+
+	if len(nodes_info) != 0 {
+		t.Fatalf("Expected empty list of nodes, got: %d", len(nodes_info))
+	}
+}

p2p/simulations/network.go

@@ -403,9 +403,8 @@ func (self *Network) getNodeByName(name string) *Node {
 func (self *Network) GetNodes() (nodes []*Node) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
-	for _, node := range self.Nodes {
+
-		nodes = append(nodes, node)
+	nodes = append(nodes, self.Nodes...)
-	}
 	return nodes
 }
 
@@ -477,7 +476,7 @@ func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) {
 	if err != nil {
 		return nil, err
 	}
-	if time.Now().Sub(conn.initiated) < dialBanTimeout {
+	if time.Since(conn.initiated) < dialBanTimeout {
 		return nil, fmt.Errorf("connection between %v and %v recently attempted", oneID, otherID)
 	}
 	if conn.Up {
@@ -502,6 +501,20 @@ func (self *Network) Shutdown() {
 	close(self.quitc)
 }
 
+//Reset resets all network properties:
+//emtpies the nodes and the connection list
+func (self *Network) Reset() {
+	self.lock.Lock()
+	defer self.lock.Unlock()
+
+	//re-initialize the maps
+	self.connMap = make(map[string]int)
+	self.nodeMap = make(map[discover.NodeID]int)
+
+	self.Nodes = nil
+	self.Conns = nil
+}
+
 // Node is a wrapper around adapters.Node which is used to track the status
 // of a node in the network
 type Node struct {
@@ -665,6 +678,12 @@ func (self *Network) Load(snap *Snapshot) error {
 		}
 	}
 	for _, conn := range snap.Conns {
+
+		if !self.GetNode(conn.One).Up || !self.GetNode(conn.Other).Up {
+			//in this case, at least one of the nodes of a connection is not up,
+			//so it would result in the snapshot `Load` to fail
+			continue
+		}
 		if err := self.Connect(conn.One, conn.Other); err != nil {
 			return err
 		}

rpc/http.go

@@ -67,7 +67,7 @@ func (hc *httpConn) Close() error {
 
 // DialHTTP creates a new RPC clients that connection to an RPC server over HTTP.
 func DialHTTP(endpoint string) (*Client, error) {
-	req, err := http.NewRequest("POST", endpoint, nil)
+	req, err := http.NewRequest(http.MethodPost, endpoint, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -149,7 +149,7 @@ func NewHTTPServer(cors []string, srv *Server) *http.Server {
 // ServeHTTP serves JSON-RPC requests over HTTP.
 func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Permit dumb empty requests for remote health-checks (AWS)
-	if r.Method == "GET" && r.ContentLength == 0 && r.URL.RawQuery == "" {
+	if r.Method == http.MethodGet && r.ContentLength == 0 && r.URL.RawQuery == "" {
 		return
 	}
 	if code, err := validateRequest(r); err != nil {
@@ -169,7 +169,7 @@ func (srv *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 // validateRequest returns a non-zero response code and error message if the
 // request is invalid.
 func validateRequest(r *http.Request) (int, error) {
-	if r.Method == "PUT" || r.Method == "DELETE" {
+	if r.Method == http.MethodPut || r.Method == http.MethodDelete {
 		return http.StatusMethodNotAllowed, errors.New("method not allowed")
 	}
 	if r.ContentLength > maxHTTPRequestContentLength {
@@ -192,7 +192,7 @@ func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler {
 
 	c := cors.New(cors.Options{
 		AllowedOrigins: allowedOrigins,
-		AllowedMethods: []string{"POST", "GET"},
+		AllowedMethods: []string{http.MethodPost, http.MethodGet},
 		MaxAge:         600,
 		AllowedHeaders: []string{"*"},
 	})

rpc/http_test.go

@@ -24,25 +24,25 @@ import (
 )
 
 func TestHTTPErrorResponseWithDelete(t *testing.T) {
-	testHTTPErrorResponse(t, "DELETE", contentType, "", http.StatusMethodNotAllowed)
+	testHTTPErrorResponse(t, http.MethodDelete, contentType, "", http.StatusMethodNotAllowed)
 }
 
 func TestHTTPErrorResponseWithPut(t *testing.T) {
-	testHTTPErrorResponse(t, "PUT", contentType, "", http.StatusMethodNotAllowed)
+	testHTTPErrorResponse(t, http.MethodPut, contentType, "", http.StatusMethodNotAllowed)
 }
 
 func TestHTTPErrorResponseWithMaxContentLength(t *testing.T) {
-	body := make([]rune, maxHTTPRequestContentLength+1, maxHTTPRequestContentLength+1)
+	body := make([]rune, maxHTTPRequestContentLength+1)
 	testHTTPErrorResponse(t,
-		"POST", contentType, string(body), http.StatusRequestEntityTooLarge)
+		http.MethodPost, contentType, string(body), http.StatusRequestEntityTooLarge)
 }
 
 func TestHTTPErrorResponseWithEmptyContentType(t *testing.T) {
-	testHTTPErrorResponse(t, "POST", "", "", http.StatusUnsupportedMediaType)
+	testHTTPErrorResponse(t, http.MethodPost, "", "", http.StatusUnsupportedMediaType)
 }
 
 func TestHTTPErrorResponseWithValidRequest(t *testing.T) {
-	testHTTPErrorResponse(t, "POST", contentType, "", 0)
+	testHTTPErrorResponse(t, http.MethodPost, contentType, "", 0)
 }
 
 func testHTTPErrorResponse(t *testing.T, method, contentType, body string, expected int) {

swarm/api/api.go

@@ -83,7 +83,7 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
 
 	// if the URI is immutable, check if the address is a hash
 	isHash := hashMatcher.MatchString(uri.Addr)
-	if uri.Immutable() {
+	if uri.Immutable() || uri.DeprecatedImmutable() {
 		if !isHash {
 			return nil, fmt.Errorf("immutable address not a content hash: %q", uri.Addr)
 		}

swarm/api/api_test.go

@@ -216,7 +216,7 @@ func TestAPIResolve(t *testing.T) {
 			api := &Api{dns: x.dns}
 			uri := &URI{Addr: x.addr, Scheme: "bzz"}
 			if x.immutable {
-				uri.Scheme = "bzzi"
+				uri.Scheme = "bzz-immutable"
 			}
 			res, err := api.Resolve(uri)
 			if err == nil {

swarm/api/client/client.go

@@ -57,7 +57,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 	if size <= 0 {
 		return "", errors.New("data size must be greater than zero")
 	}
-	req, err := http.NewRequest("POST", c.Gateway+"/bzzr:/", r)
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
 	if err != nil {
 		return "", err
 	}
@@ -79,7 +79,7 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 
 // DownloadRaw downloads raw data from swarm
 func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
-	uri := c.Gateway + "/bzzr:/" + hash
+	uri := c.Gateway + "/bzz-raw:/" + hash
 	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
 		return nil, err
@@ -269,7 +269,7 @@ func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
 //
 // where entries ending with "/" are common prefixes.
 func (c *Client) List(hash, prefix string) (*api.ManifestList, error) {
-	res, err := http.DefaultClient.Get(c.Gateway + "/bzz:/" + hash + "/" + prefix + "?list=true")
+	res, err := http.DefaultClient.Get(c.Gateway + "/bzz-list:/" + hash + "/" + prefix)
 	if err != nil {
 		return nil, err
 	}

swarm/api/config_test.go

@@ -36,7 +36,7 @@ func TestConfig(t *testing.T) {
 	one := NewDefaultConfig()
 	two := NewDefaultConfig()
 
-	if equal := reflect.DeepEqual(one, two); equal == false {
+	if equal := reflect.DeepEqual(one, two); !equal {
 		t.Fatal("Two default configs are not equal")
 	}
 

swarm/api/http/roundtripper.go

@@ -35,8 +35,8 @@ import (
 client := httpclient.New()
 // for (private) swarm proxy running locally
 client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-immutable", &http.RoundTripper{Port: port})
-client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
+client.RegisterScheme("bzz-raw", &http.RoundTripper{Port: port})
 
 The port you give the Roundtripper is the port the swarm proxy is listening on.
 If Host is left empty, localhost is assumed.

swarm/api/http/server.go

@@ -86,7 +86,7 @@ type Request struct {
 	uri *api.URI
 }
 
-// HandlePostRaw handles a POST request to a raw bzzr:/ URI, stores the request
+// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage key as a text/plain response
 func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	if r.uri.Path != "" {
@@ -290,7 +290,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
 	fmt.Fprint(w, newKey)
 }
 
-// HandleGetRaw handles a GET request to bzzr://<key> and responds with
+// HandleGetRaw handles a GET request to bzz-raw://<key> and responds with
 // the raw content stored at the given storage key
 func (s *Server) HandleGetRaw(w http.ResponseWriter, r *Request) {
 	key, err := s.api.Resolve(r.uri)
@@ -424,14 +424,13 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 	}
 }
 
-// HandleGetList handles a GET request to bzz:/<manifest>/<path> which has
+// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
-// the "list" query parameter set to "true" and returns a list of all files
+// a list of all files contained in <manifest> under <path> grouped into
-// contained in <manifest> under <path> grouped into common prefixes using
+// common prefixes using "/" as a delimiter
-// "/" as a delimiter
 func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// ensure the root path has a trailing slash so that relative URLs work
 	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/?list=true", http.StatusMovedPermanently)
+		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
@@ -453,7 +452,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
 		err := htmlListTemplate.Execute(w, &htmlListData{
-			URI:  r.uri,
+			URI: &api.URI{
+				Scheme: "bzz",
+				Addr:   r.uri.Addr,
+				Path:   r.uri.Path,
+			},
 			List: &list,
 		})
 		if err != nil {
@@ -589,7 +592,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch r.Method {
 	case "POST":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandlePostRaw(w, req)
 		} else {
 			s.HandlePostFiles(w, req)
@@ -601,7 +604,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		//   new manifest leaving the existing one intact, so it isn't
 		//   strictly a traditional PUT request which replaces content
 		//   at a URI, and POST is more ubiquitous)
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No PUT to %s allowed.", uri), http.StatusBadRequest)
 			return
 		} else {
@@ -609,28 +612,28 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 
 	case "DELETE":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			ShowError(w, r, fmt.Sprintf("No DELETE to %s allowed.", uri), http.StatusBadRequest)
 			return
 		}
 		s.HandleDelete(w, req)
 
 	case "GET":
-		if uri.Raw() {
+		if uri.Raw() || uri.DeprecatedRaw() {
 			s.HandleGetRaw(w, req)
 			return
 		}
 
+		if uri.List() {
+			s.HandleGetList(w, req)
+			return
+		}
+
 		if r.Header.Get("Accept") == "application/x-tar" {
 			s.HandleGetFiles(w, req)
 			return
 		}
 
-		if r.URL.Query().Get("list") == "true" {
-			s.HandleGetList(w, req)
-			return
-		}
-
 		s.HandleGetFile(w, req)
 
 	default:

swarm/api/http/server_test.go

@@ -70,7 +70,7 @@ func TestBzzrGetPath(t *testing.T) {
 		wg.Wait()
 	}
 
-	_, err = http.Get(srv.URL + "/bzzr:/" + common.ToHex(key[0])[2:] + "/a")
+	_, err = http.Get(srv.URL + "/bzz-raw:/" + common.ToHex(key[0])[2:] + "/a")
 	if err != nil {
 		t.Fatalf("Failed to connect to proxy: %v", err)
 	}
@@ -79,7 +79,7 @@ func TestBzzrGetPath(t *testing.T) {
 		var resp *http.Response
 		var respbody []byte
 
-		url := srv.URL + "/bzzr:/"
+		url := srv.URL + "/bzz-raw:/"
 		if k[:] != "" {
 			url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain"
 		}
@@ -104,16 +104,106 @@ func TestBzzrGetPath(t *testing.T) {
 		}
 	}
 
+	for _, c := range []struct {
+		path string
+		json string
+		html string
+	}{
+		{
+			path: "/",
+			json: `{"common_prefixes":["a/"]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"a/\">a/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/",
+			json: `{"common_prefixes":["a/b/"],"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/a","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\t<tr>\n\t  <td><a href=\"b/\">b/</a></td>\n\t  <td>DIR</td>\n\t  <td>-</td>\n\t</tr>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"a\">a</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/a/b/",
+			json: `{"entries":[{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/b","mod_time":"0001-01-01T00:00:00Z"},{"hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","path":"a/b/c","mod_time":"0001-01-01T00:00:00Z"}]}`,
+			html: "<!DOCTYPE html>\n<html>\n<head>\n  <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\">\n  <title>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</title>\n</head>\n\n<body>\n  <h1>Swarm index of bzz:/262e5c08c03c2789b6daef487dfa14b4d132f5340d781a3ecb1d5122ab65640c/a/b/</h1>\n  <hr>\n  <table>\n    <thead>\n      <tr>\n\t<th>Path</th>\n\t<th>Type</th>\n\t<th>Size</th>\n      </tr>\n    </thead>\n\n    <tbody>\n      \n\n      \n\t<tr>\n\t  <td><a href=\"b\">b</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n\t<tr>\n\t  <td><a href=\"c\">c</a></td>\n\t  <td></td>\n\t  <td>0</td>\n\t</tr>\n      \n  </table>\n  <hr>\n</body>\n",
+		},
+		{
+			path: "/x",
+		},
+		{
+			path: "",
+		},
+	} {
+		k := c.path
+		url := srv.URL + "/bzz-list:/"
+		if k[:] != "" {
+			url += common.ToHex(key[0])[2:] + "/" + k[1:]
+		}
+		t.Run("json list "+c.path, func(t *testing.T) {
+			resp, err := http.Get(url)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			body := strings.TrimSpace(string(respbody))
+			if body != c.json {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %v, got %v", k, c.json, body)
+				}
+			}
+		})
+		t.Run("html list "+c.path, func(t *testing.T) {
+			req, err := http.NewRequest(http.MethodGet, url, nil)
+			if err != nil {
+				t.Fatalf("New request: %v", err)
+			}
+			req.Header.Set("Accept", "text/html")
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("HTTP request: %v", err)
+			}
+			defer resp.Body.Close()
+			respbody, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Fatalf("Read response body: %v", err)
+			}
+
+			if string(respbody) != c.html {
+				isexpectedfailrequest := false
+
+				for _, r := range expectedfailrequests {
+					if k[:] == r {
+						isexpectedfailrequest = true
+					}
+				}
+				if !isexpectedfailrequest {
+					t.Errorf("Response list body %q does not match, expected: %q, got %q", k, c.html, string(respbody))
+				}
+			}
+		})
+	}
+
 	nonhashtests := []string{
 		srv.URL + "/bzz:/name",
-		srv.URL + "/bzzi:/nonhash",
+		srv.URL + "/bzz-immutable:/nonhash",
-		srv.URL + "/bzzr:/nonhash",
+		srv.URL + "/bzz-raw:/nonhash",
+		srv.URL + "/bzz-list:/nonhash",
 	}
 
 	nonhashresponses := []string{
 		"error resolving name: no DNS to resolve name: &#34;name&#34;",
 		"error resolving nonhash: immutable address not a content hash: &#34;nonhash&#34;",
 		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
+		"error resolving nonhash: no DNS to resolve name: &#34;nonhash&#34;",
 	}
 
 	for i, url := range nonhashtests {

swarm/api/http/templates.go

@@ -52,7 +52,7 @@ var htmlListTemplate = template.Must(template.New("html-list").Funcs(template.Fu
     <tbody>
       {{ range .List.CommonPrefixes }}
 	<tr>
-	  <td><a href="{{ basename . }}/?list=true">{{ basename . }}/</a></td>
+	  <td><a href="{{ basename . }}/">{{ basename . }}/</a></td>
 	  <td>DIR</td>
 	  <td>-</td>
 	</tr>

swarm/api/uri.go

@@ -26,7 +26,13 @@ import (
 type URI struct {
 	// Scheme has one of the following values:
 	//
-	// * bzz  - an entry in a swarm manifest
+	// * bzz           - an entry in a swarm manifest
+	// * bzz-raw       - raw swarm content
+	// * bzz-immutable - immutable URI of an entry in a swarm manifest
+	//                   (address is not resolved)
+	// * bzz-list      -  list of all files contained in a swarm manifest
+	//
+	// Deprecated Schemes:
 	// * bzzr - raw swarm content
 	// * bzzi - immutable URI of an entry in a swarm manifest
 	//          (address is not resolved)
@@ -50,7 +56,8 @@ type URI struct {
 // * <scheme>://<addr>
 // * <scheme>://<addr>/<path>
 //
-// with scheme one of bzz, bzzr or bzzi
+// with scheme one of bzz, bzz-raw, bzz-immutable or bzz-list
+// or deprecated ones bzzr and bzzi
 func Parse(rawuri string) (*URI, error) {
 	u, err := url.Parse(rawuri)
 	if err != nil {
@@ -60,7 +67,7 @@ func Parse(rawuri string) (*URI, error) {
 
 	// check the scheme is valid
 	switch uri.Scheme {
-	case "bzz", "bzzi", "bzzr":
+	case "bzz", "bzz-raw", "bzz-immutable", "bzz-list", "bzzr", "bzzi":
 	default:
 		return nil, fmt.Errorf("unknown scheme %q", u.Scheme)
 	}
@@ -84,10 +91,22 @@ func Parse(rawuri string) (*URI, error) {
 }
 
 func (u *URI) Raw() bool {
-	return u.Scheme == "bzzr"
+	return u.Scheme == "bzz-raw"
 }
 
 func (u *URI) Immutable() bool {
+	return u.Scheme == "bzz-immutable"
+}
+
+func (u *URI) List() bool {
+	return u.Scheme == "bzz-list"
+}
+
+func (u *URI) DeprecatedRaw() bool {
+	return u.Scheme == "bzzr"
+}
+
+func (u *URI) DeprecatedImmutable() bool {
 	return u.Scheme == "bzzi"
 }
 

swarm/api/uri_test.go

@@ -23,11 +23,14 @@ import (
 
 func TestParseURI(t *testing.T) {
 	type test struct {
-		uri             string
+		uri                       string
-		expectURI       *URI
+		expectURI                 *URI
-		expectErr       bool
+		expectErr                 bool
-		expectRaw       bool
+		expectRaw                 bool
-		expectImmutable bool
+		expectImmutable           bool
+		expectList                bool
+		expectDeprecatedRaw       bool
+		expectDeprecatedImmutable bool
 	}
 	tests := []test{
 		{
@@ -47,13 +50,13 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz"},
 		},
 		{
-			uri:             "bzzi:",
+			uri:             "bzz-immutable:",
-			expectURI:       &URI{Scheme: "bzzi"},
+			expectURI:       &URI{Scheme: "bzz-immutable"},
 			expectImmutable: true,
 		},
 		{
-			uri:       "bzzr:",
+			uri:       "bzz-raw:",
-			expectURI: &URI{Scheme: "bzzr"},
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
@@ -69,18 +72,18 @@ func TestParseURI(t *testing.T) {
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
 		{
-			uri:       "bzzr:/",
+			uri:       "bzz-raw:/",
-			expectURI: &URI{Scheme: "bzzr"},
+			expectURI: &URI{Scheme: "bzz-raw"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123",
+			uri:       "bzz-raw:/abc123",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123"},
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123"},
 			expectRaw: true,
 		},
 		{
-			uri:       "bzzr:/abc123/path/to/entry",
+			uri:       "bzz-raw:/abc123/path/to/entry",
-			expectURI: &URI{Scheme: "bzzr", Addr: "abc123", Path: "path/to/entry"},
+			expectURI: &URI{Scheme: "bzz-raw", Addr: "abc123", Path: "path/to/entry"},
 			expectRaw: true,
 		},
 		{
@@ -95,6 +98,36 @@ func TestParseURI(t *testing.T) {
 			uri:       "bzz://abc123/path/to/entry",
 			expectURI: &URI{Scheme: "bzz", Addr: "abc123", Path: "path/to/entry"},
 		},
+		{
+			uri:        "bzz-list:",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:        "bzz-list:/",
+			expectURI:  &URI{Scheme: "bzz-list"},
+			expectList: true,
+		},
+		{
+			uri:                 "bzzr:",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                 "bzzr:/",
+			expectURI:           &URI{Scheme: "bzzr"},
+			expectDeprecatedRaw: true,
+		},
+		{
+			uri:                       "bzzi:",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
+		{
+			uri:                       "bzzi:/",
+			expectURI:                 &URI{Scheme: "bzzi"},
+			expectDeprecatedImmutable: true,
+		},
 	}
 	for _, x := range tests {
 		actual, err := Parse(x.uri)
@@ -116,5 +149,14 @@ func TestParseURI(t *testing.T) {
 		if actual.Immutable() != x.expectImmutable {
 			t.Fatalf("expected %s immutable to be %t, got %t", x.uri, x.expectImmutable, actual.Immutable())
 		}
+		if actual.List() != x.expectList {
+			t.Fatalf("expected %s list to be %t, got %t", x.uri, x.expectList, actual.List())
+		}
+		if actual.DeprecatedRaw() != x.expectDeprecatedRaw {
+			t.Fatalf("expected %s deprecated raw to be %t, got %t", x.uri, x.expectDeprecatedRaw, actual.DeprecatedRaw())
+		}
+		if actual.DeprecatedImmutable() != x.expectDeprecatedImmutable {
+			t.Fatalf("expected %s deprecated immutable to be %t, got %t", x.uri, x.expectDeprecatedImmutable, actual.DeprecatedImmutable())
+		}
 	}
 }

swarm/dev/scripts/random-uploads.sh

@@ -46,7 +46,7 @@ main() {
 }
 
 do_random_upload() {
-  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzzr:/"
+  curl -fsSL -X POST --data-binary "$(random_data)" "http://${addr}/bzz-raw:/"
 }
 
 random_data() {

swarm/fuse/swarmfs_test.go

@@ -95,7 +95,7 @@ func mountDir(t *testing.T, api *api.Api, files map[string]fileInfo, bzzHash str
 	}
 
 	// Test listMounts
-	if found == false {
+	if !found {
 		t.Fatalf("Error getting mounts information for %v: %v", mountDir, err)
 	}
 
@@ -185,10 +185,8 @@ func isDirEmpty(name string) bool {
 	defer f.Close()
 
 	_, err = f.Readdirnames(1)
-	if err == io.EOF {
+
-		return true
+	return err == io.EOF
-	}
-	return false
 }
 
 type testAPI struct {
@@ -388,7 +386,7 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T) {
 	d.Read(contents)
 	finfo := files["1.txt"]
 
-	if bytes.Compare(finfo.contents[:6024][5000:], contents) != 0 {
+	if !bytes.Equal(finfo.contents[:6024][5000:], contents) {
 		t.Fatalf("File seek contents mismatch")
 	}
 	d.Close()

swarm/storage/chunker_test.go

@@ -77,7 +77,7 @@ func (self *chunkerTester) Split(chunker Splitter, data io.Reader, size int64, c
 
 	key, err = chunker.Split(data, size, chunkC, swg, nil)
 	if err != nil && expectedError == nil {
-		err = errors.New(fmt.Sprintf("Split error: %v", err))
+		err = fmt.Errorf("Split error: %v", err)
 	}
 
 	if chunkC != nil {
@@ -123,7 +123,7 @@ func (self *chunkerTester) Append(chunker Splitter, rootKey Key, data io.Reader,
 
 	key, err = chunker.Append(rootKey, data, chunkC, swg, nil)
 	if err != nil && expectedError == nil {
-		err = errors.New(fmt.Sprintf("Append error: %v", err))
+		err = fmt.Errorf("Append error: %v", err)
 	}
 
 	if chunkC != nil {

swarm/storage/pyramid.go

@@ -391,7 +391,7 @@ func (self *PyramidChunker) prepareChunks(isAppend bool, chunkLevel [][]*TreeEnt
 	parent := NewTreeEntry(self)
 	var unFinishedChunk *Chunk
 
-	if isAppend == true && len(chunkLevel[0]) != 0 {
+	if isAppend && len(chunkLevel[0]) != 0 {
 
 		lastIndex := len(chunkLevel[0]) - 1
 		ent := chunkLevel[0][lastIndex]
@@ -512,7 +512,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
 		}
 	}
 
-	if compress == false && last == false {
+	if !compress && !last {
 		return
 	}
 
@@ -522,7 +522,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
 	for lvl := int64(ent.level); lvl < endLvl; lvl++ {
 
 		lvlCount := int64(len(chunkLevel[lvl]))
-		if lvlCount == 1 && last == true {
+		if lvlCount == 1 && last {
 			copy(rootKey, chunkLevel[lvl][0].key)
 			return
 		}
@@ -540,7 +540,7 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
 				nextLvlCount = int64(len(chunkLevel[lvl+1]) - 1)
 				tempEntry = chunkLevel[lvl+1][nextLvlCount]
 			}
-			if isAppend == true && tempEntry != nil && tempEntry.updatePending == true {
+			if isAppend && tempEntry != nil && tempEntry.updatePending {
 				updateEntry := &TreeEntry{
 					level:         int(lvl + 1),
 					branchCount:   0,
@@ -585,9 +585,9 @@ func (self *PyramidChunker) buildTree(isAppend bool, chunkLevel [][]*TreeEntry,
 
 		}
 
-		if isAppend == false {
+		if !isAppend {
 			chunkWG.Wait()
-			if compress == true {
+			if compress {
 				chunkLevel[lvl] = nil
 			}
 		}
@@ -599,7 +599,7 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre
 	if ent != nil {
 
 		// wait for data chunks to get over before processing the tree chunk
-		if last == true {
+		if last {
 			chunkWG.Wait()
 		}
 
@@ -612,7 +612,7 @@ func (self *PyramidChunker) enqueueTreeChunk(chunkLevel [][]*TreeEntry, ent *Tre
 		}
 
 		// Update or append based on weather it is a new entry or being reused
-		if ent.updatePending == true {
+		if ent.updatePending {
 			chunkWG.Wait()
 			chunkLevel[ent.level][ent.index] = ent
 		} else {

whisper/whisperv6/whisper_test.go

@@ -80,8 +80,7 @@ func TestWhisperBasic(t *testing.T) {
 		t.Fatalf("failed w.Messages.")
 	}
 
-	var derived []byte
+	derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New)
-	derived = pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New)
 	if !validateSymmetricKey(derived) {
 		t.Fatalf("failed validateSymmetricKey with param = %v.", derived)
 	}