all: merge master to swarm-network-rewrite

.github/CODEOWNERS

@@ -5,5 +5,7 @@ accounts/usbwallet @karalabe
 consensus          @karalabe
 core/              @karalabe @holiman
 eth/               @karalabe
+les/               @zsfelfoldi
+light/             @zsfelfoldi
 mobile/            @karalabe
 p2p/               @fjl @zsfelfoldi

.travis.yml

@@ -14,17 +14,6 @@ matrix:
         - go run build/ci.go install
         - go run build/ci.go test -coverage
 
-    - os: linux
-      dist: trusty
-      sudo: required
-      go: 1.9.x
-      script:
-        - sudo modprobe fuse
-        - sudo chmod 666 /dev/fuse
-        - sudo chown root:$USER /etc/fuse.conf
-        - go run build/ci.go install
-        - go run build/ci.go test -coverage
-
     - os: osx
       go: 1.9.x
       script:

README.md

@@ -6,6 +6,7 @@ Official golang implementation of the Ethereum protocol.
 https://camo.githubusercontent.com/915b7be44ada53c290eb157634330494ebe3e30a/68747470733a2f2f676f646f632e6f72672f6769746875622e636f6d2f676f6c616e672f6764646f3f7374617475732e737667
 )](https://godoc.org/github.com/ethereum/go-ethereum)
 [![Go Report Card](https://goreportcard.com/badge/github.com/ethereum/go-ethereum)](https://goreportcard.com/report/github.com/ethereum/go-ethereum)
+[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethereum/go-ethereum?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 Automated builds are available for stable releases and the unstable master branch.

accounts/abi/argument.go

@@ -169,6 +169,21 @@ func (arguments Arguments) unpackAtomic(v interface{}, marshalledValues []interf
 	return set(elem, reflectValue, arguments.NonIndexed()[0])
 }
 
+// Computes the full size of an array;
+// i.e. counting nested arrays, which count towards size for unpacking.
+func getArraySize(arr *Type) int {
+	size := arr.Size
+	// Arrays can be nested, with each element being the same size
+	arr = arr.Elem
+	for arr.T == ArrayTy {
+		// Keep multiplying by elem.Size while the elem is an array.
+		size *= arr.Size
+		arr = arr.Elem
+	}
+	// Now we have the full array size, including its children.
+	return size
+}
+
 // UnpackValues can be used to unpack ABI-encoded hexdata according to the ABI-specification,
 // without supplying a struct to unpack into. Instead, this method returns a list containing the
 // values. An atomic argument will be a list with one element.
@@ -181,9 +196,14 @@ func (arguments Arguments) UnpackValues(data []byte) ([]interface{}, error) {
 			// If we have a static array, like [3]uint256, these are coded as
 			// just like uint256,uint256,uint256.
 			// This means that we need to add two 'virtual' arguments when
-			// we count the index from now on
+			// we count the index from now on.
-
+			//
-			virtualArgs += arg.Type.Size - 1
+			// Array values nested multiple levels deep are also encoded inline:
+			// [2][3]uint256: uint256,uint256,uint256,uint256,uint256,uint256
+			//
+			// Calculate the full array size to get the correct offset for the next argument.
+			// Decrement it by 1, as the normal index increment is still applied.
+			virtualArgs += getArraySize(&arg.Type) - 1
 		}
 		if err != nil {
 			return nil, err

accounts/abi/bind/bind.go

@@ -164,118 +164,147 @@ var bindType = map[Lang]func(kind abi.Type) string{
 	LangJava: bindTypeJava,
 }
 
+// Helper function for the binding generators.
+// It reads the unmatched characters after the inner type-match,
+//  (since the inner type is a prefix of the total type declaration),
+//  looks for valid arrays (possibly a dynamic one) wrapping the inner type,
+//  and returns the sizes of these arrays.
+//
+// Returned array sizes are in the same order as solidity signatures; inner array size first.
+// Array sizes may also be "", indicating a dynamic array.
+func wrapArray(stringKind string, innerLen int, innerMapping string) (string, []string) {
+	remainder := stringKind[innerLen:]
+	//find all the sizes
+	matches := regexp.MustCompile(`\[(\d*)\]`).FindAllStringSubmatch(remainder, -1)
+	parts := make([]string, 0, len(matches))
+	for _, match := range matches {
+		//get group 1 from the regex match
+		parts = append(parts, match[1])
+	}
+	return innerMapping, parts
+}
+
+// Translates the array sizes to a Go-lang declaration of a (nested) array of the inner type.
+// Simply returns the inner type if arraySizes is empty.
+func arrayBindingGo(inner string, arraySizes []string) string {
+	out := ""
+	//prepend all array sizes, from outer (end arraySizes) to inner (start arraySizes)
+	for i := len(arraySizes) - 1; i >= 0; i-- {
+		out += "[" + arraySizes[i] + "]"
+	}
+	out += inner
+	return out
+}
+
 // bindTypeGo converts a Solidity type to a Go one. Since there is no clear mapping
 // from all Solidity types to Go ones (e.g. uint17), those that cannot be exactly
 // mapped will use an upscaled type (e.g. *big.Int).
 func bindTypeGo(kind abi.Type) string {
 	stringKind := kind.String()
+	innerLen, innerMapping := bindUnnestedTypeGo(stringKind)
+	return arrayBindingGo(wrapArray(stringKind, innerLen, innerMapping))
+}
+
+// The inner function of bindTypeGo, this finds the inner type of stringKind.
+// (Or just the type itself if it is not an array or slice)
+// The length of the matched part is returned, with the the translated type.
+func bindUnnestedTypeGo(stringKind string) (int, string) {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
-		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		return len("address"), "common.Address"
-		if len(parts) != 2 {
-			return stringKind
-		}
-		return fmt.Sprintf("%scommon.Address", parts[1])
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
-		if len(parts) != 3 {
+		return len(parts[0]), fmt.Sprintf("[%s]byte", parts[1])
-			return stringKind
-		}
-		return fmt.Sprintf("%s[%s]byte", parts[2], parts[1])
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
-		if len(parts) != 4 {
-			return stringKind
-		}
 		switch parts[2] {
 		case "8", "16", "32", "64":
-			return fmt.Sprintf("%s%sint%s", parts[3], parts[1], parts[2])
+			return len(parts[0]), fmt.Sprintf("%sint%s", parts[1], parts[2])
 		}
-		return fmt.Sprintf("%s*big.Int", parts[3])
+		return len(parts[0]), "*big.Int"
 
-	case strings.HasPrefix(stringKind, "bool") || strings.HasPrefix(stringKind, "string"):
+	case strings.HasPrefix(stringKind, "bool"):
-		parts := regexp.MustCompile(`([a-z]+)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		return len("bool"), "bool"
-		if len(parts) != 3 {
+
-			return stringKind
+	case strings.HasPrefix(stringKind, "string"):
-		}
+		return len("string"), "string"
-		return fmt.Sprintf("%s%s", parts[2], parts[1])
 
 	default:
-		return stringKind
+		return len(stringKind), stringKind
 	}
 }
 
+// Translates the array sizes to a Java declaration of a (nested) array of the inner type.
+// Simply returns the inner type if arraySizes is empty.
+func arrayBindingJava(inner string, arraySizes []string) string {
+	// Java array type declarations do not include the length.
+	return inner + strings.Repeat("[]", len(arraySizes))
+}
+
 // bindTypeJava converts a Solidity type to a Java one. Since there is no clear mapping
 // from all Solidity types to Java ones (e.g. uint17), those that cannot be exactly
 // mapped will use an upscaled type (e.g. BigDecimal).
 func bindTypeJava(kind abi.Type) string {
 	stringKind := kind.String()
+	innerLen, innerMapping := bindUnnestedTypeJava(stringKind)
+	return arrayBindingJava(wrapArray(stringKind, innerLen, innerMapping))
+}
+
+// The inner function of bindTypeJava, this finds the inner type of stringKind.
+// (Or just the type itself if it is not an array or slice)
+// The length of the matched part is returned, with the the translated type.
+func bindUnnestedTypeJava(stringKind string) (int, string) {
 
 	switch {
 	case strings.HasPrefix(stringKind, "address"):
 		parts := regexp.MustCompile(`address(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
 		if len(parts) != 2 {
-			return stringKind
+			return len(stringKind), stringKind
 		}
 		if parts[1] == "" {
-			return fmt.Sprintf("Address")
+			return len("address"), "Address"
 		}
-		return fmt.Sprintf("Addresses")
+		return len(parts[0]), "Addresses"
 
 	case strings.HasPrefix(stringKind, "bytes"):
-		parts := regexp.MustCompile(`bytes([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		parts := regexp.MustCompile(`bytes([0-9]*)`).FindStringSubmatch(stringKind)
-		if len(parts) != 3 {
+		if len(parts) != 2 {
-			return stringKind
+			return len(stringKind), stringKind
 		}
-		if parts[2] != "" {
+		return len(parts[0]), "byte[]"
-			return "byte[][]"
-		}
-		return "byte[]"
 
 	case strings.HasPrefix(stringKind, "int") || strings.HasPrefix(stringKind, "uint"):
-		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		//Note that uint and int (without digits) are also matched,
-		if len(parts) != 4 {
+		// these are size 256, and will translate to BigInt (the default).
-			return stringKind
+		parts := regexp.MustCompile(`(u)?int([0-9]*)`).FindStringSubmatch(stringKind)
+		if len(parts) != 3 {
+			return len(stringKind), stringKind
 		}
-		switch parts[2] {
+
-		case "8", "16", "32", "64":
+		namedSize := map[string]string{
-			if parts[1] == "" {
+			"8":  "byte",
-				if parts[3] == "" {
+			"16": "short",
-					return fmt.Sprintf("int%s", parts[2])
+			"32": "int",
-				}
+			"64": "long",
-				return fmt.Sprintf("int%s[]", parts[2])
+		}[parts[2]]
-			}
+
+		//default to BigInt
+		if namedSize == "" {
+			namedSize = "BigInt"
 		}
-		if parts[3] == "" {
+		return len(parts[0]), namedSize
-			return fmt.Sprintf("BigInt")
-		}
-		return fmt.Sprintf("BigInts")
 
 	case strings.HasPrefix(stringKind, "bool"):
-		parts := regexp.MustCompile(`bool(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		return len("bool"), "boolean"
-		if len(parts) != 2 {
-			return stringKind
-		}
-		if parts[1] == "" {
-			return fmt.Sprintf("bool")
-		}
-		return fmt.Sprintf("bool[]")
 
 	case strings.HasPrefix(stringKind, "string"):
-		parts := regexp.MustCompile(`string(\[[0-9]*\])?`).FindStringSubmatch(stringKind)
+		return len("string"), "String"
-		if len(parts) != 2 {
-			return stringKind
-		}
-		if parts[1] == "" {
-			return fmt.Sprintf("String")
-		}
-		return fmt.Sprintf("String[]")
 
 	default:
-		return stringKind
+		return len(stringKind), stringKind
 	}
 }
 
@@ -325,11 +354,13 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 		return "String"
 	case "string[]":
 		return "Strings"
-	case "bool":
+	case "boolean":
 		return "Bool"
-	case "bool[]":
+	case "boolean[]":
 		return "Bools"
-	case "BigInt":
+	case "BigInt[]":
+		return "BigInts"
+	default:
 		parts := regexp.MustCompile(`(u)?int([0-9]*)(\[[0-9]*\])?`).FindStringSubmatch(solKind.String())
 		if len(parts) != 4 {
 			return javaKind
@@ -344,8 +375,6 @@ func namedTypeJava(javaKind string, solKind abi.Type) string {
 		default:
 			return javaKind
 		}
-	default:
-		return javaKind
 	}
 }
 

accounts/abi/bind/bind_test.go

@@ -737,6 +737,72 @@ var bindTests = []struct {
 			}
 		`,
 	},
+	{
+		`DeeplyNestedArray`,
+		`
+			contract DeeplyNestedArray {
+				uint64[3][4][5] public deepUint64Array;
+				function storeDeepUintArray(uint64[3][4][5] arr) public {
+					deepUint64Array = arr;
+				}
+				function retrieveDeepArray() public view returns (uint64[3][4][5]) {
+					return deepUint64Array;
+				}
+			}
+		`,
+		`6060604052341561000f57600080fd5b6106438061001e6000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff168063344248551461005c5780638ed4573a1461011457806398ed1856146101ab575b600080fd5b341561006757600080fd5b610112600480806107800190600580602002604051908101604052809291906000905b828210156101055783826101800201600480602002604051908101604052809291906000905b828210156100f25783826060020160038060200260405190810160405280929190826003602002808284378201915050505050815260200190600101906100b0565b505050508152602001906001019061008a565b5050505091905050610208565b005b341561011f57600080fd5b61012761021d565b604051808260056000925b8184101561019b578284602002015160046000925b8184101561018d5782846020020151600360200280838360005b8381101561017c578082015181840152602081019050610161565b505050509050019260010192610147565b925050509260010192610132565b9250505091505060405180910390f35b34156101b657600080fd5b6101de6004808035906020019091908035906020019091908035906020019091905050610309565b604051808267ffffffffffffffff1667ffffffffffffffff16815260200191505060405180910390f35b80600090600561021992919061035f565b5050565b6102256103b0565b6000600580602002604051908101604052809291906000905b8282101561030057838260040201600480602002604051908101604052809291906000905b828210156102ed578382016003806020026040519081016040528092919082600380156102d9576020028201916000905b82829054906101000a900467ffffffffffffffff1667ffffffffffffffff16815260200190600801906020826007010492830192600103820291508084116102945790505b505050505081526020019060010190610263565b505050508152602001906001019061023e565b50505050905090565b60008360058110151561031857fe5b600402018260048110151561032957fe5b018160038110151561033757fe5b6004918282040191900660080292509250509054906101000a900467ffffffffffffffff1681565b826005600402810192821561039f579160200282015b8281111561039e5782518290600461038e9291906103df565b5091602001919060040190610375565b5b5090506103ac919061042d565b5090565b610780604051908101604052806005905b6103c9610459565b8152602001906001900390816103c15790505090565b826004810192821561041c579160200282015b8281111561041b5782518290600361040b929190610488565b50916020019190600101906103f2565b5b5090506104299190610536565b5090565b61045691905b8082111561045257600081816104499190610562565b50600401610433565b5090565b90565b610180604051908101604052806004905b6104726105a7565b81526020019060019003908161046a5790505090565b82600380016004900481019282156105255791602002820160005b838211156104ef57835183826101000a81548167ffffffffffffffff021916908367ffffffffffffffff16021790555092602001926008016020816007010492830192600103026104a3565b80156105235782816101000a81549067ffffffffffffffff02191690556008016020816007010492830192600103026104ef565b505b50905061053291906105d9565b5090565b61055f91905b8082111561055b57600081816105529190610610565b5060010161053c565b5090565b90565b50600081816105719190610610565b50600101600081816105839190610610565b50600101600081816105959190610610565b5060010160006105a59190610610565b565b6060604051908101604052806003905b600067ffffffffffffffff168152602001906001900390816105b75790505090565b61060d91905b8082111561060957600081816101000a81549067ffffffffffffffff0219169055506001016105df565b5090565b90565b50600090555600a165627a7a7230582087e5a43f6965ab6ef7a4ff056ab80ed78fd8c15cff57715a1bf34ec76a93661c0029`,
+		`[{"constant":false,"inputs":[{"name":"arr","type":"uint64[3][4][5]"}],"name":"storeDeepUintArray","outputs":[],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":true,"inputs":[],"name":"retrieveDeepArray","outputs":[{"name":"","type":"uint64[3][4][5]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[{"name":"","type":"uint256"},{"name":"","type":"uint256"},{"name":"","type":"uint256"}],"name":"deepUint64Array","outputs":[{"name":"","type":"uint64"}],"payable":false,"stateMutability":"view","type":"function"}]`,
+		`
+			// Generate a new random account and a funded simulator
+			key, _ := crypto.GenerateKey()
+			auth := bind.NewKeyedTransactor(key)
+			sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}})
+
+			//deploy the test contract
+			_, _, testContract, err := DeployDeeplyNestedArray(auth, sim)
+			if err != nil {
+				t.Fatalf("Failed to deploy test contract: %v", err)
+			}
+
+			// Finish deploy.
+			sim.Commit()
+
+			//Create coordinate-filled array, for testing purposes.
+			testArr := [5][4][3]uint64{}
+			for i := 0; i < 5; i++ {
+				testArr[i] = [4][3]uint64{}
+				for j := 0; j < 4; j++ {
+					testArr[i][j] = [3]uint64{}
+					for k := 0; k < 3; k++ {
+						//pack the coordinates, each array value will be unique, and can be validated easily.
+						testArr[i][j][k] = uint64(i) << 16 | uint64(j) << 8 | uint64(k)
+					}
+				}
+			}
+
+			if _, err := testContract.StoreDeepUintArray(&bind.TransactOpts{
+				From: auth.From,
+				Signer: auth.Signer,
+			}, testArr); err != nil {
+				t.Fatalf("Failed to store nested array in test contract: %v", err)
+			}
+
+			sim.Commit()
+
+			retrievedArr, err := testContract.RetrieveDeepArray(&bind.CallOpts{
+				From: auth.From,
+				Pending: false,
+			})
+			if err != nil {
+				t.Fatalf("Failed to retrieve nested array from test contract: %v", err)
+			}
+
+			//quick check to see if contents were copied
+			// (See accounts/abi/unpack_test.go for more extensive testing)
+			if retrievedArr[4][3][2] != testArr[4][3][2] {
+				t.Fatalf("Retrieved value does not match expected value! got: %d, expected: %d. %v", retrievedArr[4][3][2], testArr[4][3][2], err)
+			}`,
+	},
 }
 
 // Tests that packages generated by the binder can be successfully compiled and

accounts/abi/pack_test.go

@@ -299,6 +299,11 @@ func TestPack(t *testing.T) {
 			[32]byte{1},
 			common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000"),
 		},
+		{
+			"uint32[2][3][4]",
+			[4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}},
+			common.Hex2Bytes("000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018"),
+		},
 		{
 			"address[]",
 			[]common.Address{{1}, {2}},

accounts/abi/unpack.go

@@ -93,6 +93,17 @@ func readFixedBytes(t Type, word []byte) (interface{}, error) {
 
 }
 
+func getFullElemSize(elem *Type) int {
+	//all other should be counted as 32 (slices have pointers to respective elements)
+	size := 32
+	//arrays wrap it, each element being the same size
+	for elem.T == ArrayTy {
+		size *= elem.Size
+		elem = elem.Elem
+	}
+	return size
+}
+
 // iteratively unpack elements
 func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) {
 	if size < 0 {
@@ -104,7 +115,6 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 
 	// this value will become our slice or our array, depending on the type
 	var refSlice reflect.Value
-	slice := output[start : start+size*32]
 
 	if t.T == SliceTy {
 		// declare our slice
@@ -116,15 +126,20 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 		return nil, fmt.Errorf("abi: invalid type in array/slice unpacking stage")
 	}
 
-	for i, j := start, 0; j*32 < len(slice); i, j = i+32, j+1 {
+	// Arrays have packed elements, resulting in longer unpack steps.
-		// this corrects the arrangement so that we get all the underlying array values
+	// Slices have just 32 bytes per element (pointing to the contents).
-		if t.Elem.T == ArrayTy && j != 0 {
+	elemSize := 32
-			i = start + t.Elem.Size*32*j
+	if t.T == ArrayTy {
-		}
+		elemSize = getFullElemSize(t.Elem)
+	}
+
+	for i, j := start, 0; j < size; i, j = i+elemSize, j+1 {
+
 		inter, err := toGoType(i, *t.Elem, output)
 		if err != nil {
 			return nil, err
 		}
+
 		// append the item to our reflect slice
 		refSlice.Index(j).Set(reflect.ValueOf(inter))
 	}

accounts/abi/unpack_test.go

@@ -189,6 +189,11 @@ var unpackTests = []unpackTest{
 		enc:  "00000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
 		want: [2]uint32{1, 2},
 	},
+	{
+		def:  `[{"type": "uint32[2][3][4]"}]`,
+		enc:  "000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000000050000000000000000000000000000000000000000000000000000000000000006000000000000000000000000000000000000000000000000000000000000000700000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000009000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000b000000000000000000000000000000000000000000000000000000000000000c000000000000000000000000000000000000000000000000000000000000000d000000000000000000000000000000000000000000000000000000000000000e000000000000000000000000000000000000000000000000000000000000000f000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000110000000000000000000000000000000000000000000000000000000000000012000000000000000000000000000000000000000000000000000000000000001300000000000000000000000000000000000000000000000000000000000000140000000000000000000000000000000000000000000000000000000000000015000000000000000000000000000000000000000000000000000000000000001600000000000000000000000000000000000000000000000000000000000000170000000000000000000000000000000000000000000000000000000000000018",
+		want: [4][3][2]uint32{{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}, {{13, 14}, {15, 16}, {17, 18}}, {{19, 20}, {21, 22}, {23, 24}}},
+	},
 	{
 		def:  `[{"type": "uint64[]"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002",
@@ -435,6 +440,46 @@ func TestMultiReturnWithArray(t *testing.T) {
 	}
 }
 
+func TestMultiReturnWithDeeplyNestedArray(t *testing.T) {
+	// Similar to TestMultiReturnWithArray, but with a special case in mind:
+	//  values of nested static arrays count towards the size as well, and any element following
+	//  after such nested array argument should be read with the correct offset,
+	//  so that it does not read content from the previous array argument.
+	const definition = `[{"name" : "multi", "outputs": [{"type": "uint64[3][2][4]"}, {"type": "uint64"}]}]`
+	abi, err := JSON(strings.NewReader(definition))
+	if err != nil {
+		t.Fatal(err)
+	}
+	buff := new(bytes.Buffer)
+	// construct the test array, each 3 char element is joined with 61 '0' chars,
+	// to from the ((3 + 61) * 0.5) = 32 byte elements in the array.
+	buff.Write(common.Hex2Bytes(strings.Join([]string{
+		"", //empty, to apply the 61-char separator to the first element as well.
+		"111", "112", "113", "121", "122", "123",
+		"211", "212", "213", "221", "222", "223",
+		"311", "312", "313", "321", "322", "323",
+		"411", "412", "413", "421", "422", "423",
+	}, "0000000000000000000000000000000000000000000000000000000000000")))
+	buff.Write(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000009876"))
+
+	ret1, ret1Exp := new([4][2][3]uint64), [4][2][3]uint64{
+		{{0x111, 0x112, 0x113}, {0x121, 0x122, 0x123}},
+		{{0x211, 0x212, 0x213}, {0x221, 0x222, 0x223}},
+		{{0x311, 0x312, 0x313}, {0x321, 0x322, 0x323}},
+		{{0x411, 0x412, 0x413}, {0x421, 0x422, 0x423}},
+	}
+	ret2, ret2Exp := new(uint64), uint64(0x9876)
+	if err := abi.Unpack(&[]interface{}{ret1, ret2}, "multi", buff.Bytes()); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(*ret1, ret1Exp) {
+		t.Error("array result", *ret1, "!= Expected", ret1Exp)
+	}
+	if *ret2 != ret2Exp {
+		t.Error("int result", *ret2, "!= Expected", ret2Exp)
+	}
+}
+
 func TestUnmarshal(t *testing.T) {
 	const definition = `[
 	{ "name" : "int", "constant" : false, "outputs": [ { "type": "uint256" } ] },

build/ci.go

@@ -182,13 +182,13 @@ func doInstall(cmdline []string) {
 	// Check Go version. People regularly open issues about compilation
 	// failure with outdated Go. This should save them the trouble.
 	if !strings.Contains(runtime.Version(), "devel") {
-		// Figure out the minor version number since we can't textually compare (1.10 < 1.7)
+		// Figure out the minor version number since we can't textually compare (1.10 < 1.8)
 		var minor int
 		fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor)
 
-		if minor < 7 {
+		if minor < 8 {
 			log.Println("You have Go version", runtime.Version())
-			log.Println("go-ethereum requires at least Go version 1.7 and cannot")
+			log.Println("go-ethereum requires at least Go version 1.8 and cannot")
 			log.Println("be compiled with an earlier version. Please upgrade your Go installation.")
 			os.Exit(1)
 		}

cmd/faucet/faucet.go

@@ -533,9 +533,11 @@ func (f *faucet) loop() {
 	}
 	defer sub.Unsubscribe()
 
-	for {
+	// Start a goroutine to update the state from head notifications in the background
-		select {
+	update := make(chan *types.Header)
-		case head := <-heads:
+
+	go func() {
+		for head := range update {
 			// New chain head arrived, query the current stats and stream to clients
 			var (
 				balance *big.Int
@@ -588,6 +590,17 @@ func (f *faucet) loop() {
 				}
 			}
 			f.lock.RUnlock()
+		}
+	}()
+	// Wait for various events and assing to the appropriate background threads
+	for {
+		select {
+		case head := <-heads:
+			// New head arrived, send if for state update if there's none running
+			select {
+			case update <- head:
+			default:
+			}
 
 		case <-f.update:
 			// Pending requests updated, stream to clients

cmd/puppeth/genesis.go

@@ -168,19 +168,18 @@ type parityChainSpec struct {
 	Engine struct {
 		Ethash struct {
 			Params struct {
-				MinimumDifficulty      *hexutil.Big   `json:"minimumDifficulty"`
+				MinimumDifficulty      *hexutil.Big `json:"minimumDifficulty"`
-				DifficultyBoundDivisor *hexutil.Big   `json:"difficultyBoundDivisor"`
+				DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"`
-				GasLimitBoundDivisor   hexutil.Uint64 `json:"gasLimitBoundDivisor"`
+				DurationLimit          *hexutil.Big `json:"durationLimit"`
-				DurationLimit          *hexutil.Big   `json:"durationLimit"`
+				BlockReward            *hexutil.Big `json:"blockReward"`
-				BlockReward            *hexutil.Big   `json:"blockReward"`
+				HomesteadTransition    uint64       `json:"homesteadTransition"`
-				HomesteadTransition    uint64         `json:"homesteadTransition"`
+				EIP150Transition       uint64       `json:"eip150Transition"`
-				EIP150Transition       uint64         `json:"eip150Transition"`
+				EIP160Transition       uint64       `json:"eip160Transition"`
-				EIP160Transition       uint64         `json:"eip160Transition"`
+				EIP161abcTransition    uint64       `json:"eip161abcTransition"`
-				EIP161abcTransition    uint64         `json:"eip161abcTransition"`
+				EIP161dTransition      uint64       `json:"eip161dTransition"`
-				EIP161dTransition      uint64         `json:"eip161dTransition"`
+				EIP649Reward           *hexutil.Big `json:"eip649Reward"`
-				EIP649Reward           *hexutil.Big   `json:"eip649Reward"`
+				EIP100bTransition      uint64       `json:"eip100bTransition"`
-				EIP100bTransition      uint64         `json:"eip100bTransition"`
+				EIP649Transition       uint64       `json:"eip649Transition"`
-				EIP649Transition       uint64         `json:"eip649Transition"`
 			} `json:"params"`
 		} `json:"Ethash"`
 	} `json:"engine"`
@@ -188,6 +187,7 @@ type parityChainSpec struct {
 	Params struct {
 		MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"`
 		MinGasLimit          hexutil.Uint64 `json:"minGasLimit"`
+		GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"`
 		NetworkID            hexutil.Uint64 `json:"networkID"`
 		MaxCodeSize          uint64         `json:"maxCodeSize"`
 		EIP155Transition     uint64         `json:"eip155Transition"`
@@ -270,7 +270,6 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 	}
 	spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty)
 	spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor)
-	spec.Engine.Ethash.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
 	spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit)
 	spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward)
 	spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64()
@@ -284,6 +283,7 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin
 
 	spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize)
 	spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit)
+	spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor)
 	spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainId.Uint64())
 	spec.Params.MaxCodeSize = params.MaxCodeSize
 	spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64()

cmd/swarm/manifest.go

@@ -35,7 +35,7 @@ const bzzManifestJSON = "application/bzz-manifest+json"
 func add(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 3 {
-		utils.Fatalf("Need atleast three arguments <MHASH> <path> <HASH> [<content-type>]")
+		utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH> [<content-type>]")
 	}
 
 	var (
@@ -69,7 +69,7 @@ func update(ctx *cli.Context) {
 
 	args := ctx.Args()
 	if len(args) < 3 {
-		utils.Fatalf("Need atleast three arguments <MHASH> <path> <HASH>")
+		utils.Fatalf("Need at least three arguments <MHASH> <path> <HASH>")
 	}
 
 	var (
@@ -101,7 +101,7 @@ func update(ctx *cli.Context) {
 func remove(ctx *cli.Context) {
 	args := ctx.Args()
 	if len(args) < 2 {
-		utils.Fatalf("Need atleast two arguments <MHASH> <path>")
+		utils.Fatalf("Need at least two arguments <MHASH> <path>")
 	}
 
 	var (

cmd/wnode/main.go

@@ -22,6 +22,7 @@ package main
 import (
 	"bufio"
 	"crypto/ecdsa"
+	crand "crypto/rand"
 	"crypto/sha512"
 	"encoding/binary"
 	"encoding/hex"
@@ -48,6 +49,7 @@ import (
 )
 
 const quitCommand = "~Q"
+const entropySize = 32
 
 // singletons
 var (
@@ -55,6 +57,7 @@ var (
 	shh        *whisper.Whisper
 	done       chan struct{}
 	mailServer mailserver.WMailServer
+	entropy    [entropySize]byte
 
 	input = bufio.NewReader(os.Stdin)
 )
@@ -76,14 +79,15 @@ var (
 
 // cmd arguments
 var (
-	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't actively connect to peers, wait for incoming connections")
+	bootstrapMode  = flag.Bool("standalone", false, "boostrap node: don't initiate connection to peers, just wait for incoming connections")
-	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither send nor decrypt messages")
+	forwarderMode  = flag.Bool("forwarder", false, "forwarder mode: only forward messages, neither encrypt nor decrypt messages")
 	mailServerMode = flag.Bool("mailserver", false, "mail server mode: delivers expired messages on demand")
 	requestMail    = flag.Bool("mailclient", false, "request expired messages from the bootstrap server")
 	asymmetricMode = flag.Bool("asym", false, "use asymmetric encryption")
 	generateKey    = flag.Bool("generatekey", false, "generate and show the private key")
 	fileExMode     = flag.Bool("fileexchange", false, "file exchange mode")
-	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics")
+	fileReader     = flag.Bool("filereader", false, "load and decrypt messages saved as files, display as plain text")
+	testMode       = flag.Bool("test", false, "use of predefined parameters for diagnostics (password, etc.)")
 	echoMode       = flag.Bool("echo", false, "echo mode: prints some arguments for diagnostics")
 
 	argVerbosity = flag.Int("verbosity", int(log.LvlError), "log verbosity level")
@@ -99,13 +103,14 @@ var (
 	argIDFile  = flag.String("idfile", "", "file name with node id (private key)")
 	argEnode   = flag.String("boot", "", "bootstrap node you want to connect to (e.g. enode://e454......08d50@52.176.211.200:16428)")
 	argTopic   = flag.String("topic", "", "topic in hexadecimal format (e.g. 70a4beef)")
-	argSaveDir = flag.String("savedir", "", "directory where incoming messages will be saved as files")
+	argSaveDir = flag.String("savedir", "", "directory where all incoming messages will be saved as files")
 )
 
 func main() {
 	processArgs()
 	initialize()
 	run()
+	shutdown()
 }
 
 func processArgs() {
@@ -205,21 +210,6 @@ func initialize() {
 		MinimumAcceptedPOW: *argPoW,
 	}
 
-	if *mailServerMode {
-		if len(msPassword) == 0 {
-			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
-			if err != nil {
-				utils.Fatalf("Failed to read Mail Server password: %s", err)
-			}
-		}
-
-		shh = whisper.New(cfg)
-		shh.RegisterServer(&mailServer)
-		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
-	} else {
-		shh = whisper.New(cfg)
-	}
-
 	if *argPoW != whisper.DefaultMinimumPoW {
 		err := shh.SetMinimumPoW(*argPoW)
 		if err != nil {
@@ -261,6 +251,26 @@ func initialize() {
 		maxPeers = 800
 	}
 
+	_, err = crand.Read(entropy[:])
+	if err != nil {
+		utils.Fatalf("crypto/rand failed: %s", err)
+	}
+
+	if *mailServerMode {
+		if len(msPassword) == 0 {
+			msPassword, err = console.Stdin.PromptPassword("Please enter the Mail Server password: ")
+			if err != nil {
+				utils.Fatalf("Failed to read Mail Server password: %s", err)
+			}
+		}
+
+		shh = whisper.New(cfg)
+		shh.RegisterServer(&mailServer)
+		mailServer.Init(shh, *argDBPath, msPassword, *argServerPoW)
+	} else {
+		shh = whisper.New(cfg)
+	}
+
 	server = &p2p.Server{
 		Config: p2p.Config{
 			PrivateKey:     nodeid,
@@ -276,10 +286,11 @@ func initialize() {
 	}
 }
 
-func startServer() {
+func startServer() error {
 	err := server.Start()
 	if err != nil {
-		utils.Fatalf("Failed to start Whisper peer: %s.", err)
+		fmt.Printf("Failed to start Whisper peer: %s.", err)
+		return err
 	}
 
 	fmt.Printf("my public key: %s \n", common.ToHex(crypto.FromECDSAPub(&asymKey.PublicKey)))
@@ -298,6 +309,7 @@ func startServer() {
 	if !*forwarderMode {
 		fmt.Printf("Please type the message. To quit type: '%s'\n", quitCommand)
 	}
+	return nil
 }
 
 func isKeyValid(k *ecdsa.PublicKey) bool {
@@ -411,8 +423,10 @@ func waitForConnection(timeout bool) {
 }
 
 func run() {
-	defer mailServer.Close()
+	err := startServer()
-	startServer()
+	if err != nil {
+		return
+	}
 	defer server.Stop()
 	shh.Start(nil)
 	defer shh.Stop()
@@ -425,21 +439,26 @@ func run() {
 		requestExpiredMessagesLoop()
 	} else if *fileExMode {
 		sendFilesLoop()
+	} else if *fileReader {
+		fileReaderLoop()
 	} else {
 		sendLoop()
 	}
 }
 
+func shutdown() {
+	close(done)
+	mailServer.Close()
+}
+
 func sendLoop() {
 	for {
 		s := scanLine("")
 		if s == quitCommand {
 			fmt.Println("Quit command received")
-			close(done)
+			return
-			break
 		}
 		sendMsg([]byte(s))
-
 		if *asymmetricMode {
 			// print your own message for convenience,
 			// because in asymmetric mode it is impossible to decrypt it
@@ -455,13 +474,11 @@ func sendFilesLoop() {
 		s := scanLine("")
 		if s == quitCommand {
 			fmt.Println("Quit command received")
-			close(done)
+			return
-			break
 		}
 		b, err := ioutil.ReadFile(s)
 		if err != nil {
 			fmt.Printf(">>> Error: %s \n", err)
-			continue
 		} else {
 			h := sendMsg(b)
 			if (h == common.Hash{}) {
@@ -475,6 +492,38 @@ func sendFilesLoop() {
 	}
 }
 
+func fileReaderLoop() {
+	watcher1 := shh.GetFilter(symFilterID)
+	watcher2 := shh.GetFilter(asymFilterID)
+	if watcher1 == nil && watcher2 == nil {
+		fmt.Println("Error: neither symmetric nor asymmetric filter is installed")
+		return
+	}
+
+	for {
+		s := scanLine("")
+		if s == quitCommand {
+			fmt.Println("Quit command received")
+			return
+		}
+		raw, err := ioutil.ReadFile(s)
+		if err != nil {
+			fmt.Printf(">>> Error: %s \n", err)
+		} else {
+			env := whisper.Envelope{Data: raw} // the topic is zero
+			msg := env.Open(watcher1)          // force-open envelope regardless of the topic
+			if msg == nil {
+				msg = env.Open(watcher2)
+			}
+			if msg == nil {
+				fmt.Printf(">>> Error: failed to decrypt the message \n")
+			} else {
+				printMessageInfo(msg)
+			}
+		}
+	}
+}
+
 func scanLine(prompt string) string {
 	if len(prompt) > 0 {
 		fmt.Print(prompt)
@@ -548,20 +597,18 @@ func messageLoop() {
 	for {
 		select {
 		case <-ticker.C:
-			messages := sf.Retrieve()
+			m1 := sf.Retrieve()
+			m2 := af.Retrieve()
+			messages := append(m1, m2...)
 			for _, msg := range messages {
-				if *fileExMode || len(msg.Payload) > 2048 {
+				// All messages are saved upon specifying argSaveDir.
+				// fileExMode only specifies how messages are displayed on the console after they are saved.
+				// if fileExMode == true, only the hashes are displayed, since messages might be too big.
+				if len(*argSaveDir) > 0 {
 					writeMessageToFile(*argSaveDir, msg)
-				} else {
-					printMessageInfo(msg)
 				}
-			}
 
-			messages = af.Retrieve()
+				if !*fileExMode && len(msg.Payload) <= 2048 {
-			for _, msg := range messages {
-				if *fileExMode || len(msg.Payload) > 2048 {
-					writeMessageToFile(*argSaveDir, msg)
-				} else {
 					printMessageInfo(msg)
 				}
 			}
@@ -596,27 +643,30 @@ func writeMessageToFile(dir string, msg *whisper.ReceivedMessage) {
 		address = crypto.PubkeyToAddress(*msg.Src)
 	}
 
-	if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
+	// this is a sample code; uncomment if you don't want to save your own messages.
-		// message from myself: don't save, only report
+	//if whisper.IsPubKeyEqual(msg.Src, &asymKey.PublicKey) {
-		fmt.Printf("\n%s <%x>: message received: '%s'\n", timestamp, address, name)
+	//	fmt.Printf("\n%s <%x>: message from myself received, not saved: '%s'\n", timestamp, address, name)
-	} else if len(dir) > 0 {
+	//	return
+	//}
+
+	if len(dir) > 0 {
 		fullpath := filepath.Join(dir, name)
-		err := ioutil.WriteFile(fullpath, msg.Payload, 0644)
+		err := ioutil.WriteFile(fullpath, msg.Raw, 0644)
 		if err != nil {
 			fmt.Printf("\n%s {%x}: message received but not saved: %s\n", timestamp, address, err)
 		} else {
-			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Payload))
+			fmt.Printf("\n%s {%x}: message received and saved as '%s' (%d bytes)\n", timestamp, address, name, len(msg.Raw))
 		}
 	} else {
-		fmt.Printf("\n%s {%x}: big message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Payload), name)
+		fmt.Printf("\n%s {%x}: message received (%d bytes), but not saved: %s\n", timestamp, address, len(msg.Raw), name)
 	}
 }
 
 func requestExpiredMessagesLoop() {
-	var key, peerID []byte
+	var key, peerID, bloom []byte
 	var timeLow, timeUpp uint32
 	var t string
-	var xt, empty whisper.TopicType
+	var xt whisper.TopicType
 
 	keyID, err := shh.AddSymKeyFromPassword(msPassword)
 	if err != nil {
@@ -639,18 +689,19 @@ func requestExpiredMessagesLoop() {
 				utils.Fatalf("Failed to parse the topic: %s", err)
 			}
 			xt = whisper.BytesToTopic(x)
+			bloom = whisper.TopicToBloom(xt)
+			obfuscateBloom(bloom)
+		} else {
+			bloom = whisper.MakeFullNodeBloom()
 		}
 		if timeUpp == 0 {
 			timeUpp = 0xFFFFFFFF
 		}
 
-		data := make([]byte, 8+whisper.TopicLength)
+		data := make([]byte, 8, 8+whisper.BloomFilterSize)
 		binary.BigEndian.PutUint32(data, timeLow)
 		binary.BigEndian.PutUint32(data[4:], timeUpp)
-		copy(data[8:], xt[:])
+		data = append(data, bloom...)
-		if xt == empty {
-			data = data[:8]
-		}
 
 		var params whisper.MessageParams
 		params.PoW = *argServerPoW
@@ -684,3 +735,20 @@ func extractIDFromEnode(s string) []byte {
 	}
 	return n.ID[:]
 }
+
+// obfuscateBloom adds 16 random bits to the the bloom
+// filter, in order to obfuscate the containing topics.
+// it does so deterministically within every session.
+// despite additional bits, it will match on average
+// 32000 times less messages than full node's bloom filter.
+func obfuscateBloom(bloom []byte) {
+	const half = entropySize / 2
+	for i := 0; i < half; i++ {
+		x := int(entropy[i])
+		if entropy[half+i] < 128 {
+			x += 256
+		}
+
+		bloom[x/8] = 1 << uint(x%8) // set the bit number X
+	}
+}

consensus/ethash/algorithm.go

@@ -19,6 +19,7 @@ package ethash
 import (
 	"encoding/binary"
 	"hash"
+	"math/big"
 	"reflect"
 	"runtime"
 	"sync"
@@ -47,6 +48,48 @@ const (
 	loopAccesses       = 64      // Number of accesses in hashimoto loop
 )
 
+// cacheSize returns the size of the ethash verification cache that belongs to a certain
+// block number.
+func cacheSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return cacheSizes[epoch]
+	}
+	return calcCacheSize(epoch)
+}
+
+// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcCacheSize(epoch int) uint64 {
+	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
+	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * hashBytes
+	}
+	return size
+}
+
+// datasetSize returns the size of the ethash mining dataset that belongs to a certain
+// block number.
+func datasetSize(block uint64) uint64 {
+	epoch := int(block / epochLength)
+	if epoch < maxEpoch {
+		return datasetSizes[epoch]
+	}
+	return calcDatasetSize(epoch)
+}
+
+// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
+// however, we always take the highest prime below the linearly growing threshold in order
+// to reduce the risk of accidental regularities leading to cyclic behavior.
+func calcDatasetSize(epoch int) uint64 {
+	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
+	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
+		size -= 2 * mixBytes
+	}
+	return size
+}
+
 // hasher is a repetitive hasher allowing the same hash data structures to be
 // reused between hash runs instead of requiring new ones to be created.
 type hasher func(dest []byte, data []byte)

consensus/ethash/algorithm_go1.7.go

@@ -1,47 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// +build !go1.8
-
-package ethash
-
-// cacheSize calculates and returns the size of the ethash verification cache that
-// belongs to a certain block number. The cache size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func cacheSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}
-
-// datasetSize calculates and returns the size of the ethash mining dataset that
-// belongs to a certain block number. The dataset size grows linearly, however, we
-// always take the highest prime below the linearly growing threshold in order to
-// reduce the risk of accidental regularities leading to cyclic behavior.
-func datasetSize(block uint64) uint64 {
-	// If we have a pre-generated value, use that
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	// We don't have a way to verify primes fast before Go 1.8
-	panic("fast prime testing unsupported in Go < 1.8")
-}

consensus/ethash/algorithm_go1.8.go

@@ -1,63 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// +build go1.8
-
-package ethash
-
-import "math/big"
-
-// cacheSize returns the size of the ethash verification cache that belongs to a certain
-// block number.
-func cacheSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return cacheSizes[epoch]
-	}
-	return calcCacheSize(epoch)
-}
-
-// calcCacheSize calculates the cache size for epoch. The cache size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcCacheSize(epoch int) uint64 {
-	size := cacheInitBytes + cacheGrowthBytes*uint64(epoch) - hashBytes
-	for !new(big.Int).SetUint64(size / hashBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * hashBytes
-	}
-	return size
-}
-
-// datasetSize returns the size of the ethash mining dataset that belongs to a certain
-// block number.
-func datasetSize(block uint64) uint64 {
-	epoch := int(block / epochLength)
-	if epoch < maxEpoch {
-		return datasetSizes[epoch]
-	}
-	return calcDatasetSize(epoch)
-}
-
-// calcDatasetSize calculates the dataset size for epoch. The dataset size grows linearly,
-// however, we always take the highest prime below the linearly growing threshold in order
-// to reduce the risk of accidental regularities leading to cyclic behavior.
-func calcDatasetSize(epoch int) uint64 {
-	size := datasetInitBytes + datasetGrowthBytes*uint64(epoch) - mixBytes
-	for !new(big.Int).SetUint64(size / mixBytes).ProbablyPrime(1) { // Always accurate for n < 2^64
-		size -= 2 * mixBytes
-	}
-	return size
-}

consensus/ethash/algorithm_go1.8_test.go

@@ -1,37 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// +build go1.8
-
-package ethash
-
-import "testing"
-
-// Tests whether the dataset size calculator works correctly by cross checking the
-// hard coded lookup table with the value generated by it.
-func TestSizeCalculations(t *testing.T) {
-	// Verify all the cache and dataset sizes from the lookup table.
-	for epoch, want := range cacheSizes {
-		if size := calcCacheSize(epoch); size != want {
-			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-	for epoch, want := range datasetSizes {
-		if size := calcDatasetSize(epoch); size != want {
-			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
-		}
-	}
-}

consensus/ethash/algorithm_test.go

@@ -30,6 +30,22 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 )
 
+// Tests whether the dataset size calculator works correctly by cross checking the
+// hard coded lookup table with the value generated by it.
+func TestSizeCalculations(t *testing.T) {
+	// Verify all the cache and dataset sizes from the lookup table.
+	for epoch, want := range cacheSizes {
+		if size := calcCacheSize(epoch); size != want {
+			t.Errorf("cache %d: cache size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+	for epoch, want := range datasetSizes {
+		if size := calcDatasetSize(epoch); size != want {
+			t.Errorf("dataset %d: dataset size mismatch: have %d, want %d", epoch, size, want)
+		}
+	}
+}
+
 // Tests that verification caches can be correctly generated.
 func TestCacheGeneration(t *testing.T) {
 	tests := []struct {

consensus/ethash/consensus.go

@@ -53,7 +53,6 @@ var (
 	errDuplicateUncle    = errors.New("duplicate uncle")
 	errUncleIsAncestor   = errors.New("uncle is ancestor")
 	errDanglingUncle     = errors.New("uncle's parent is not ancestor")
-	errNonceOutOfRange   = errors.New("nonce out of range")
 	errInvalidDifficulty = errors.New("non-positive difficulty")
 	errInvalidMixDigest  = errors.New("invalid mix digest")
 	errInvalidPoW        = errors.New("invalid proof-of-work")
@@ -356,7 +355,7 @@ func calcDifficultyByzantium(time uint64, parent *types.Header) *big.Int {
 	if x.Cmp(params.MinimumDifficulty) < 0 {
 		x.Set(params.MinimumDifficulty)
 	}
-	// calculate a fake block numer for the ice-age delay:
+	// calculate a fake block number for the ice-age delay:
 	//   https://github.com/ethereum/EIPs/pull/669
 	//   fake_block_number = min(0, block.number - 3_000_000
 	fakeBlockNumber := new(big.Int)
@@ -474,18 +473,13 @@ func (ethash *Ethash) VerifySeal(chain consensus.ChainReader, header *types.Head
 	if ethash.shared != nil {
 		return ethash.shared.VerifySeal(chain, header)
 	}
-	// Sanity check that the block number is below the lookup table size (60M blocks)
-	number := header.Number.Uint64()
-	if number/epochLength >= maxEpoch {
-		// Go < 1.7 cannot calculate new cache/dataset sizes (no fast prime check)
-		return errNonceOutOfRange
-	}
 	// Ensure that we have a valid difficulty for the block
 	if header.Difficulty.Sign() <= 0 {
 		return errInvalidDifficulty
 	}
-
 	// Recompute the digest and PoW value and verify against the header
+	number := header.Number.Uint64()
+
 	cache := ethash.cache(number)
 	size := datasetSize(number)
 	if ethash.config.PowMode == ModeTest {

core/asm/lexer.go

@@ -206,7 +206,7 @@ func lexLine(l *lexer) stateFn {
 			return lexComment
 		case isSpace(r):
 			l.ignore()
-		case isAlphaNumeric(r) || r == '_':
+		case isLetter(r) || r == '_':
 			return lexElement
 		case isNumber(r):
 			return lexNumber
@@ -278,7 +278,7 @@ func lexElement(l *lexer) stateFn {
 	return lexLine
 }
 
-func isAlphaNumeric(t rune) bool {
+func isLetter(t rune) bool {
 	return unicode.IsLetter(t)
 }
 

core/database_util.go

@@ -47,6 +47,7 @@ var (
 	headHeaderKey = []byte("LastHeader")
 	headBlockKey  = []byte("LastBlock")
 	headFastKey   = []byte("LastFast")
+	trieSyncKey   = []byte("TrieSync")
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`).
 	headerPrefix        = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
@@ -146,6 +147,16 @@ func GetHeadFastBlockHash(db DatabaseReader) common.Hash {
 	return common.BytesToHash(data)
 }
 
+// GetTrieSyncProgress retrieves the number of tries nodes fast synced to allow
+// reportinc correct numbers across restarts.
+func GetTrieSyncProgress(db DatabaseReader) uint64 {
+	data, _ := db.Get(trieSyncKey)
+	if len(data) == 0 {
+		return 0
+	}
+	return new(big.Int).SetBytes(data).Uint64()
+}
+
 // GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil
 // if the header's not found.
 func GetHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue {
@@ -374,6 +385,15 @@ func WriteHeadFastBlockHash(db ethdb.Putter, hash common.Hash) error {
 	return nil
 }
 
+// WriteTrieSyncProgress stores the fast sync trie process counter to support
+// retrieving it across restarts.
+func WriteTrieSyncProgress(db ethdb.Putter, count uint64) error {
+	if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil {
+		log.Crit("Failed to store fast sync trie progress", "err", err)
+	}
+	return nil
+}
+
 // WriteHeader serializes a block header into the database.
 func WriteHeader(db ethdb.Putter, header *types.Header) error {
 	data, err := rlp.EncodeToBytes(header)

eth/downloader/downloader.go

@@ -27,6 +27,7 @@ import (
 
 	ethereum "github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
@@ -221,7 +222,10 @@ func New(mode SyncMode, stateDb ethdb.Database, mux *event.TypeMux, chain BlockC
 		quitCh:         make(chan struct{}),
 		stateCh:        make(chan dataPack),
 		stateSyncStart: make(chan *stateSync),
-		trackStateReq:  make(chan *stateReq),
+		syncStatsState: stateSyncStats{
+			processed: core.GetTrieSyncProgress(stateDb),
+		},
+		trackStateReq: make(chan *stateReq),
 	}
 	go dl.qosTuner()
 	go dl.stateFetcher()

eth/downloader/statesync.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -466,4 +467,7 @@ func (s *stateSync) updateStats(written, duplicate, unexpected int, duration tim
 	if written > 0 || duplicate > 0 || unexpected > 0 {
 		log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "retry", len(s.tasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected)
 	}
+	if written > 0 {
+		core.WriteTrieSyncProgress(s.d.stateDB, s.d.syncStatsState.processed)
+	}
 }

eth/handler.go

@@ -249,7 +249,8 @@ func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *p
 // handle is the callback invoked to manage the life cycle of an eth peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 	p.Log().Debug("Ethereum peer connected", "name", p.Name())

internal/debug/api.go

@@ -140,10 +140,9 @@ func (h *HandlerT) GoTrace(file string, nsec uint) error {
 	return nil
 }
 
-// BlockProfile turns on CPU profiling for nsec seconds and writes
+// BlockProfile turns on goroutine profiling for nsec seconds and writes profile data to
-// profile data to file. It uses a profile rate of 1 for most accurate
+// file. It uses a profile rate of 1 for most accurate information. If a different rate is
-// information. If a different rate is desired, set the rate
+// desired, set the rate and write the profile manually.
-// and write the profile manually.
 func (*HandlerT) BlockProfile(file string, nsec uint) error {
 	runtime.SetBlockProfileRate(1)
 	time.Sleep(time.Duration(nsec) * time.Second)
@@ -162,6 +161,26 @@ func (*HandlerT) WriteBlockProfile(file string) error {
 	return writeProfile("block", file)
 }
 
+// MutexProfile turns on mutex profiling for nsec seconds and writes profile data to file.
+// It uses a profile rate of 1 for most accurate information. If a different rate is
+// desired, set the rate and write the profile manually.
+func (*HandlerT) MutexProfile(file string, nsec uint) error {
+	runtime.SetMutexProfileFraction(1)
+	time.Sleep(time.Duration(nsec) * time.Second)
+	defer runtime.SetMutexProfileFraction(0)
+	return writeProfile("mutex", file)
+}
+
+// SetMutexProfileFraction sets the rate of mutex profiling.
+func (*HandlerT) SetMutexProfileFraction(rate int) {
+	runtime.SetMutexProfileFraction(rate)
+}
+
+// WriteMutexProfile writes a goroutine blocking profile to the given file.
+func (*HandlerT) WriteMutexProfile(file string) error {
+	return writeProfile("mutex", file)
+}
+
 // WriteMemProfile writes an allocation profile to the given file.
 // Note that the profiling rate cannot be set through the API,
 // it must be set on the command line.

internal/ethapi/api.go

@@ -1035,14 +1035,14 @@ func (s *PublicTransactionPoolAPI) GetRawTransactionByHash(ctx context.Context,
 func (s *PublicTransactionPoolAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
 	tx, blockHash, blockNumber, index := core.GetTransaction(s.b.ChainDb(), hash)
 	if tx == nil {
-		return nil, errors.New("unknown transaction")
+		return nil, nil
 	}
 	receipts, err := s.b.GetReceipts(ctx, blockHash)
 	if err != nil {
 		return nil, err
 	}
 	if len(receipts) <= int(index) {
-		return nil, errors.New("unknown receipt")
+		return nil, nil
 	}
 	receipt := receipts[index]
 

internal/web3ext/web3ext.go

@@ -307,6 +307,21 @@ web3._extend({
 			call: 'debug_writeBlockProfile',
 			params: 1
 		}),
+		new web3._extend.Method({
+			name: 'mutexProfile',
+			call: 'debug_mutexProfile',
+			params: 2
+		}),
+		new web3._extend.Method({
+			name: 'setMutexProfileRate',
+			call: 'debug_setMutexProfileRate',
+			params: 1
+		}),
+		new web3._extend.Method({
+			name: 'writeMutexProfile',
+			call: 'debug_writeMutexProfile',
+			params: 1
+		}),
 		new web3._extend.Method({
 			name: 'writeMemProfile',
 			call: 'debug_writeMemProfile',

les/handler.go

@@ -260,7 +260,8 @@ func (pm *ProtocolManager) newPeer(pv int, nv uint64, p *p2p.Peer, rw p2p.MsgRea
 // handle is the callback invoked to manage the life cycle of a les peer. When
 // this function terminates, the peer is disconnected.
 func (pm *ProtocolManager) handle(p *peer) error {
-	if pm.peers.Len() >= pm.maxPeers {
+	// Ignore maxPeers if this is a trusted peer
+	if pm.peers.Len() >= pm.maxPeers && !p.Peer.Info().Network.Trusted {
 		return p2p.DiscTooManyPeers
 	}
 

light/postprocess.go

@@ -58,18 +58,18 @@ type trustedCheckpoint struct {
 var (
 	mainnetCheckpoint = trustedCheckpoint{
 		name:          "mainnet",
-		sectionIdx:    153,
+		sectionIdx:    157,
-		sectionHead:   common.HexToHash("04c2114a8cbe49ba5c37a03cc4b4b8d3adfc0bd2c78e0e726405dd84afca1d63"),
+		sectionHead:   common.HexToHash("1963c080887ca7f406c2bb114293eea83e54f783f94df24b447f7e3b6317c747"),
-		chtRoot:       common.HexToHash("d7ec603e5d30b567a6e894ee7704e4603232f206d3e5a589794cec0c57bf318e"),
+		chtRoot:       common.HexToHash("42abc436567dfb678a38fa6a9f881aa4c8a4cc8eaa2def08359292c3d0bd48ec"),
-		bloomTrieRoot: common.HexToHash("0b139b8fb692e21f663ff200da287192201c28ef5813c1ac6ba02a0a4799eef9"),
+		bloomTrieRoot: common.HexToHash("281c9f8fb3cb8b37ae45e9907ef8f3b19cd22c54e297c2d6c09c1db1593dce42"),
 	}
 
 	ropstenCheckpoint = trustedCheckpoint{
 		name:          "ropsten",
-		sectionIdx:    79,
+		sectionIdx:    83,
-		sectionHead:   common.HexToHash("1b1ba890510e06411fdee9bb64ca7705c56a1a4ce3559ddb34b3680c526cb419"),
+		sectionHead:   common.HexToHash("3ca623586bc0da35f1fc8d9b6b55950f3b1f69be9c6501846a2df672adb61236"),
-		chtRoot:       common.HexToHash("71d60207af74e5a22a3e1cfbfc89f9944f91b49aa980c86fba94d568369eaf44"),
+		chtRoot:       common.HexToHash("8f08ec7783969768c6ef06e5fe3398223cbf4ae2907b676da7b6fe6c7f55b059"),
-		bloomTrieRoot: common.HexToHash("70aca4b3b6d08dde8704c95cedb1420394453c1aec390947751e69ff8c436360"),
+		bloomTrieRoot: common.HexToHash("02d86d3c6a87f8f8a92c2a59bbba2132ff6f9f61b0915a5dc28a9d8279219fd0"),
 	}
 )
 

metrics/metrics_test.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"sync"
 	"testing"
+	"time"
 )
 
 const FANOUT = 128
@@ -114,7 +115,7 @@ func Example() {
 
 	// Threadsafe registration
 	t := GetOrRegisterTimer("db.get.latency", nil)
-	t.Time(func() {})
+	t.Time(func() { time.Sleep(10 * time.Millisecond) })
 	t.Update(1)
 
 	fmt.Println(c.Count())

metrics/timer_test.go

@@ -47,8 +47,8 @@ func TestTimerStop(t *testing.T) {
 func TestTimerFunc(t *testing.T) {
 	tm := NewTimer()
 	tm.Time(func() { time.Sleep(50e6) })
-	if max := tm.Max(); 45e6 > max || max > 55e6 {
+	if max := tm.Max(); 35e6 > max || max > 95e6 {
-		t.Errorf("tm.Max(): 45e6 > %v || %v > 55e6\n", max, max)
+		t.Errorf("tm.Max(): 35e6 > %v || %v > 95e6\n", max, max)
 	}
 }
 

node/api.go

@@ -308,6 +308,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 		// Fill the counter with the metric details, formatting if requested
 		if raw {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"AvgRate01Min": metric.Rate1(),
@@ -338,6 +343,11 @@ func (api *PublicDebugAPI) Metrics(raw bool) (map[string]interface{}, error) {
 			}
 		} else {
 			switch metric := metric.(type) {
+			case metrics.Counter:
+				root[name] = map[string]interface{}{
+					"Overall": float64(metric.Count()),
+				}
+
 			case metrics.Meter:
 				root[name] = map[string]interface{}{
 					"Avg01Min": format(metric.Rate1()*60, metric.Rate1()),

p2p/rlpx.go

@@ -121,10 +121,6 @@ func (t *rlpx) close(err error) {
 	t.fd.Close()
 }
 
-// doEncHandshake runs the protocol handshake using authenticated
-// messages. the protocol handshake is the first authenticated message
-// and also verifies whether the encryption handshake 'worked' and the
-// remote side actually provided the right public key.
 func (t *rlpx) doProtoHandshake(our *protoHandshake) (their *protoHandshake, err error) {
 	// Writing our handshake happens concurrently, we prefer
 	// returning the handshake read error. If the remote side
@@ -175,6 +171,10 @@ func readProtocolHandshake(rw MsgReader, our *protoHandshake) (*protoHandshake,
 	return &hs, nil
 }
 
+// doEncHandshake runs the protocol handshake using authenticated
+// messages. the protocol handshake is the first authenticated message
+// and also verifies whether the encryption handshake 'worked' and the
+// remote side actually provided the right public key.
 func (t *rlpx) doEncHandshake(prv *ecdsa.PrivateKey, dial *discover.Node) (discover.NodeID, error) {
 	var (
 		sec secrets

swarm/api/http/error.go

@@ -116,7 +116,7 @@ func ValidateCaseErrors(r *Request) string {
 func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
 	msg := ""
 	if list.Entries == nil {
-		Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
+		Respond(w, req, "Could not resolve", http.StatusInternalServerError)
 		return
 	}
 	//make links relative
@@ -133,7 +133,6 @@ func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestL
 		//create clickable link for each entry
 		msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
 	}
-
 	Respond(w, req, msg, http.StatusMultipleChoices)
 }
 
@@ -142,7 +141,6 @@ func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestL
 //The function just takes a string message which will be displayed in the error page.
 //The code is used to evaluate which template will be displayed
 //(and return the correct HTTP status code)
-
 func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
 	additionalMessage := ValidateCaseErrors(req)
 	switch code {

whisper/mailserver/mailserver.go

@@ -17,7 +17,6 @@
 package mailserver
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 
@@ -108,17 +107,16 @@ func (s *WMailServer) DeliverMail(peer *whisper.Peer, request *whisper.Envelope)
 		return
 	}
 
-	ok, lower, upper, topic := s.validateRequest(peer.ID(), request)
+	ok, lower, upper, bloom := s.validateRequest(peer.ID(), request)
 	if ok {
-		s.processRequest(peer, lower, upper, topic)
+		s.processRequest(peer, lower, upper, bloom)
 	}
 }
 
-func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, topic whisper.TopicType) []*whisper.Envelope {
+func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, bloom []byte) []*whisper.Envelope {
 	ret := make([]*whisper.Envelope, 0)
 	var err error
 	var zero common.Hash
-	var empty whisper.TopicType
 	kl := NewDbKey(lower, zero)
 	ku := NewDbKey(upper, zero)
 	i := s.db.NewIterator(&util.Range{Start: kl.raw, Limit: ku.raw}, nil)
@@ -131,7 +129,7 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to
 			log.Error(fmt.Sprintf("RLP decoding failed: %s", err))
 		}
 
-		if topic == empty || envelope.Topic == topic {
+		if whisper.BloomFilterMatch(bloom, envelope.Bloom()) {
 			if peer == nil {
 				// used for test purposes
 				ret = append(ret, &envelope)
@@ -153,39 +151,45 @@ func (s *WMailServer) processRequest(peer *whisper.Peer, lower, upper uint32, to
 	return ret
 }
 
-func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, whisper.TopicType) {
+func (s *WMailServer) validateRequest(peerID []byte, request *whisper.Envelope) (bool, uint32, uint32, []byte) {
-	var topic whisper.TopicType
 	if s.pow > 0.0 && request.PoW() < s.pow {
-		return false, 0, 0, topic
+		return false, 0, 0, nil
 	}
 
 	f := whisper.Filter{KeySym: s.key}
 	decrypted := request.Open(&f)
 	if decrypted == nil {
 		log.Warn(fmt.Sprintf("Failed to decrypt p2p request"))
-		return false, 0, 0, topic
+		return false, 0, 0, nil
-	}
-
-	if len(decrypted.Payload) < 8 {
-		log.Warn(fmt.Sprintf("Undersized p2p request"))
-		return false, 0, 0, topic
 	}
 
 	src := crypto.FromECDSAPub(decrypted.Src)
 	if len(src)-len(peerID) == 1 {
 		src = src[1:]
 	}
-	if !bytes.Equal(peerID, src) {
+
+	// if you want to check the signature, you can do it here. e.g.:
+	// if !bytes.Equal(peerID, src) {
+	if src == nil {
 		log.Warn(fmt.Sprintf("Wrong signature of p2p request"))
-		return false, 0, 0, topic
+		return false, 0, 0, nil
+	}
+
+	var bloom []byte
+	payloadSize := len(decrypted.Payload)
+	if payloadSize < 8 {
+		log.Warn(fmt.Sprintf("Undersized p2p request"))
+		return false, 0, 0, nil
+	} else if payloadSize == 8 {
+		bloom = whisper.MakeFullNodeBloom()
+	} else if payloadSize < 8+whisper.BloomFilterSize {
+		log.Warn(fmt.Sprintf("Undersized bloom filter in p2p request"))
+		return false, 0, 0, nil
+	} else {
+		bloom = decrypted.Payload[8 : 8+whisper.BloomFilterSize]
 	}
 
 	lower := binary.BigEndian.Uint32(decrypted.Payload[:4])
 	upper := binary.BigEndian.Uint32(decrypted.Payload[4:8])
-
+	return true, lower, upper, bloom
-	if len(decrypted.Payload) >= 8+whisper.TopicLength {
-		topic = whisper.BytesToTopic(decrypted.Payload[8:])
-	}
-
-	return true, lower, upper, topic
 }

whisper/mailserver/server_test.go

@@ -17,6 +17,7 @@
 package mailserver
 
 import (
+	"bytes"
 	"crypto/ecdsa"
 	"encoding/binary"
 	"io/ioutil"
@@ -61,7 +62,7 @@ func generateEnvelope(t *testing.T) *whisper.Envelope {
 	h := crypto.Keccak256Hash([]byte("test sample data"))
 	params := &whisper.MessageParams{
 		KeySym:   h[:],
-		Topic:    whisper.TopicType{},
+		Topic:    whisper.TopicType{0x1F, 0x7E, 0xA1, 0x7F},
 		Payload:  []byte("test payload"),
 		PoW:      powRequirement,
 		WorkTime: 2,
@@ -121,6 +122,7 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) {
 		upp:   birth + 1,
 		key:   testPeerID,
 	}
+
 	singleRequest(t, server, env, p, true)
 
 	p.low, p.upp = birth+1, 0xffffffff
@@ -131,14 +133,14 @@ func deliverTest(t *testing.T, server *WMailServer, env *whisper.Envelope) {
 
 	p.low = birth - 1
 	p.upp = birth + 1
-	p.topic[0]++
+	p.topic[0] = 0xFF
 	singleRequest(t, server, env, p, false)
 }
 
 func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *ServerTestParams, expect bool) {
 	request := createRequest(t, p)
 	src := crypto.FromECDSAPub(&p.key.PublicKey)
-	ok, lower, upper, topic := server.validateRequest(src, request)
+	ok, lower, upper, bloom := server.validateRequest(src, request)
 	if !ok {
 		t.Fatalf("request validation failed, seed: %d.", seed)
 	}
@@ -148,12 +150,13 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *
 	if upper != p.upp {
 		t.Fatalf("request validation failed (upper bound), seed: %d.", seed)
 	}
-	if topic != p.topic {
+	expectedBloom := whisper.TopicToBloom(p.topic)
+	if !bytes.Equal(bloom, expectedBloom) {
 		t.Fatalf("request validation failed (topic), seed: %d.", seed)
 	}
 
 	var exist bool
-	mail := server.processRequest(nil, p.low, p.upp, p.topic)
+	mail := server.processRequest(nil, p.low, p.upp, bloom)
 	for _, msg := range mail {
 		if msg.Hash() == env.Hash() {
 			exist = true
@@ -166,17 +169,19 @@ func singleRequest(t *testing.T, server *WMailServer, env *whisper.Envelope, p *
 	}
 
 	src[0]++
-	ok, lower, upper, topic = server.validateRequest(src, request)
+	ok, lower, upper, bloom = server.validateRequest(src, request)
-	if ok {
+	if !ok {
-		t.Fatalf("request validation false positive, seed: %d (lower: %d, upper: %d).", seed, lower, upper)
+		// request should be valid regardless of signature
+		t.Fatalf("request validation false negative, seed: %d (lower: %d, upper: %d).", seed, lower, upper)
 	}
 }
 
 func createRequest(t *testing.T, p *ServerTestParams) *whisper.Envelope {
-	data := make([]byte, 8+whisper.TopicLength)
+	bloom := whisper.TopicToBloom(p.topic)
+	data := make([]byte, 8)
 	binary.BigEndian.PutUint32(data, p.low)
 	binary.BigEndian.PutUint32(data[4:], p.upp)
-	copy(data[8:], p.topic[:])
+	data = append(data, bloom...)
 
 	key, err := shh.GetSymKey(keyID)
 	if err != nil {

whisper/whisperv6/doc.go

@@ -60,7 +60,7 @@ const (
 	aesKeyLength    = 32 // in bytes
 	aesNonceLength  = 12 // in bytes; for more info please see cipher.gcmStandardNonceSize & aesgcm.NonceSize()
 	keyIDSize       = 32 // in bytes
-	bloomFilterSize = 64 // in bytes
+	BloomFilterSize = 64 // in bytes
 	flagsLength     = 1
 
 	EnvelopeHeaderLength = 20

whisper/whisperv6/envelope.go

@@ -208,6 +208,10 @@ func (e *Envelope) OpenSymmetric(key []byte) (msg *ReceivedMessage, err error) {
 
 // Open tries to decrypt an envelope, and populates the message fields in case of success.
 func (e *Envelope) Open(watcher *Filter) (msg *ReceivedMessage) {
+	if watcher == nil {
+		return nil
+	}
+
 	// The API interface forbids filters doing both symmetric and asymmetric encryption.
 	if watcher.expectsAsymmetricEncryption() && watcher.expectsSymmetricEncryption() {
 		return nil
@@ -249,7 +253,7 @@ func (e *Envelope) Bloom() []byte {
 
 // TopicToBloom converts the topic (4 bytes) to the bloom filter (64 bytes)
 func TopicToBloom(topic TopicType) []byte {
-	b := make([]byte, bloomFilterSize)
+	b := make([]byte, BloomFilterSize)
 	var index [3]int
 	for j := 0; j < 3; j++ {
 		index[j] = int(topic[j])

whisper/whisperv6/filter.go

@@ -35,6 +35,7 @@ type Filter struct {
 	PoW        float64           // Proof of work as described in the Whisper spec
 	AllowP2P   bool              // Indicates whether this filter is interested in direct peer-to-peer messages
 	SymKeyHash common.Hash       // The Keccak256Hash of the symmetric key, needed for optimization
+	id         string            // unique identifier
 
 	Messages map[common.Hash]*ReceivedMessage
 	mutex    sync.RWMutex
@@ -43,15 +44,21 @@ type Filter struct {
 // Filters represents a collection of filters
 type Filters struct {
 	watchers map[string]*Filter
-	whisper  *Whisper
+
-	mutex    sync.RWMutex
+	topicMatcher     map[TopicType]map[*Filter]struct{} // map a topic to the filters that are interested in being notified when a message matches that topic
+	allTopicsMatcher map[*Filter]struct{}               // list all the filters that will be notified of a new message, no matter what its topic is
+
+	whisper *Whisper
+	mutex   sync.RWMutex
 }
 
 // NewFilters returns a newly created filter collection
 func NewFilters(w *Whisper) *Filters {
 	return &Filters{
-		watchers: make(map[string]*Filter),
+		watchers:         make(map[string]*Filter),
-		whisper:  w,
+		topicMatcher:     make(map[TopicType]map[*Filter]struct{}),
+		allTopicsMatcher: make(map[*Filter]struct{}),
+		whisper:          w,
 	}
 }
 
@@ -81,7 +88,9 @@ func (fs *Filters) Install(watcher *Filter) (string, error) {
 		watcher.SymKeyHash = crypto.Keccak256Hash(watcher.KeySym)
 	}
 
+	watcher.id = id
 	fs.watchers[id] = watcher
+	fs.addTopicMatcher(watcher)
 	return id, err
 }
 
@@ -91,12 +100,51 @@ func (fs *Filters) Uninstall(id string) bool {
 	fs.mutex.Lock()
 	defer fs.mutex.Unlock()
 	if fs.watchers[id] != nil {
+		fs.removeFromTopicMatchers(fs.watchers[id])
 		delete(fs.watchers, id)
 		return true
 	}
 	return false
 }
 
+// addTopicMatcher adds a filter to the topic matchers.
+// If the filter's Topics array is empty, it will be tried on every topic.
+// Otherwise, it will be tried on the topics specified.
+func (fs *Filters) addTopicMatcher(watcher *Filter) {
+	if len(watcher.Topics) == 0 {
+		fs.allTopicsMatcher[watcher] = struct{}{}
+	} else {
+		for _, t := range watcher.Topics {
+			topic := BytesToTopic(t)
+			if fs.topicMatcher[topic] == nil {
+				fs.topicMatcher[topic] = make(map[*Filter]struct{})
+			}
+			fs.topicMatcher[topic][watcher] = struct{}{}
+		}
+	}
+}
+
+// removeFromTopicMatchers removes a filter from the topic matchers
+func (fs *Filters) removeFromTopicMatchers(watcher *Filter) {
+	delete(fs.allTopicsMatcher, watcher)
+	for _, topic := range watcher.Topics {
+		delete(fs.topicMatcher[BytesToTopic(topic)], watcher)
+	}
+}
+
+// getWatchersByTopic returns a slice containing the filters that
+// match a specific topic
+func (fs *Filters) getWatchersByTopic(topic TopicType) []*Filter {
+	res := make([]*Filter, 0, len(fs.allTopicsMatcher))
+	for watcher := range fs.allTopicsMatcher {
+		res = append(res, watcher)
+	}
+	for watcher := range fs.topicMatcher[topic] {
+		res = append(res, watcher)
+	}
+	return res
+}
+
 // Get returns a filter from the collection with a specific ID
 func (fs *Filters) Get(id string) *Filter {
 	fs.mutex.RLock()
@@ -112,11 +160,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
 	fs.mutex.RLock()
 	defer fs.mutex.RUnlock()
 
-	i := -1 // only used for logging info
+	candidates := fs.getWatchersByTopic(env.Topic)
-	for _, watcher := range fs.watchers {
+	for _, watcher := range candidates {
-		i++
 		if p2pMessage && !watcher.AllowP2P {
-			log.Trace(fmt.Sprintf("msg [%x], filter [%d]: p2p messages are not allowed", env.Hash(), i))
+			log.Trace(fmt.Sprintf("msg [%x], filter [%s]: p2p messages are not allowed", env.Hash(), watcher.id))
 			continue
 		}
 
@@ -128,10 +175,10 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
 			if match {
 				msg = env.Open(watcher)
 				if msg == nil {
-					log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", i)
+					log.Trace("processing message: failed to open", "message", env.Hash().Hex(), "filter", watcher.id)
 				}
 			} else {
-				log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", i)
+				log.Trace("processing message: does not match", "message", env.Hash().Hex(), "filter", watcher.id)
 			}
 		}
 
@@ -144,20 +191,6 @@ func (fs *Filters) NotifyWatchers(env *Envelope, p2pMessage bool) {
 	}
 }
 
-func (f *Filter) processEnvelope(env *Envelope) *ReceivedMessage {
-	if f.MatchEnvelope(env) {
-		msg := env.Open(f)
-		if msg != nil {
-			return msg
-		}
-
-		log.Trace("processing envelope: failed to open", "hash", env.Hash().Hex())
-	} else {
-		log.Trace("processing envelope: does not match", "hash", env.Hash().Hex())
-	}
-	return nil
-}
-
 func (f *Filter) expectsAsymmetricEncryption() bool {
 	return f.KeyAsym != nil
 }
@@ -194,16 +227,17 @@ func (f *Filter) Retrieve() (all []*ReceivedMessage) {
 
 // MatchMessage checks if the filter matches an already decrypted
 // message (i.e. a Message that has already been handled by
-// MatchEnvelope when checked by a previous filter)
+// MatchEnvelope when checked by a previous filter).
+// Topics are not checked here, since this is done by topic matchers.
 func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 	if f.PoW > 0 && msg.PoW < f.PoW {
 		return false
 	}
 
 	if f.expectsAsymmetricEncryption() && msg.isAsymmetricEncryption() {
-		return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst) && f.MatchTopic(msg.Topic)
+		return IsPubKeyEqual(&f.KeyAsym.PublicKey, msg.Dst)
 	} else if f.expectsSymmetricEncryption() && msg.isSymmetricEncryption() {
-		return f.SymKeyHash == msg.SymKeyHash && f.MatchTopic(msg.Topic)
+		return f.SymKeyHash == msg.SymKeyHash
 	}
 	return false
 }
@@ -211,27 +245,9 @@ func (f *Filter) MatchMessage(msg *ReceivedMessage) bool {
 // MatchEnvelope checks if it's worth decrypting the message. If
 // it returns `true`, client code is expected to attempt decrypting
 // the message and subsequently call MatchMessage.
+// Topics are not checked here, since this is done by topic matchers.
 func (f *Filter) MatchEnvelope(envelope *Envelope) bool {
-	if f.PoW > 0 && envelope.pow < f.PoW {
+	return f.PoW <= 0 || envelope.pow >= f.PoW
-		return false
-	}
-
-	return f.MatchTopic(envelope.Topic)
-}
-
-// MatchTopic checks that the filter captures a given topic.
-func (f *Filter) MatchTopic(topic TopicType) bool {
-	if len(f.Topics) == 0 {
-		// any topic matches
-		return true
-	}
-
-	for _, bt := range f.Topics {
-		if matchSingleTopic(topic, bt) {
-			return true
-		}
-	}
-	return false
 }
 
 func matchSingleTopic(topic TopicType, bt []byte) bool {

whisper/whisperv6/filter_test.go

@@ -303,9 +303,8 @@ func TestMatchEnvelope(t *testing.T) {
 		t.Fatalf("failed generateMessageParams with seed %d: %s.", seed, err)
 	}
 
-	params.Topic[0] = 0xFF // ensure mismatch
+	params.Topic[0] = 0xFF // topic mismatch
 
-	// mismatch with pseudo-random data
 	msg, err := NewSentMessage(params)
 	if err != nil {
 		t.Fatalf("failed to create new message with seed %d: %s.", seed, err)
@@ -314,14 +313,6 @@ func TestMatchEnvelope(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed Wrap with seed %d: %s.", seed, err)
 	}
-	match := fsym.MatchEnvelope(env)
-	if match {
-		t.Fatalf("failed MatchEnvelope symmetric with seed %d.", seed)
-	}
-	match = fasym.MatchEnvelope(env)
-	if match {
-		t.Fatalf("failed MatchEnvelope asymmetric with seed %d.", seed)
-	}
 
 	// encrypt symmetrically
 	i := mrand.Int() % 4
@@ -337,7 +328,7 @@ func TestMatchEnvelope(t *testing.T) {
 	}
 
 	// symmetric + matching topic: match
-	match = fsym.MatchEnvelope(env)
+	match := fsym.MatchEnvelope(env)
 	if !match {
 		t.Fatalf("failed MatchEnvelope() symmetric with seed %d.", seed)
 	}
@@ -396,7 +387,7 @@ func TestMatchEnvelope(t *testing.T) {
 	// asymmetric + matching topic: match
 	fasym.Topics[i] = fasym.Topics[i+1]
 	match = fasym.MatchEnvelope(env)
-	if match {
+	if !match {
 		t.Fatalf("failed MatchEnvelope(asymmetric + matching topic) with seed %d.", seed)
 	}
 
@@ -431,7 +422,8 @@ func TestMatchEnvelope(t *testing.T) {
 	// filter with topic + envelope without topic: mismatch
 	fasym.Topics = fsym.Topics
 	match = fasym.MatchEnvelope(env)
-	if match {
+	if !match {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(filter without topic + envelope without topic) with seed %d.", seed)
 	}
 }
@@ -487,7 +479,8 @@ func TestMatchMessageSym(t *testing.T) {
 
 	// topic mismatch
 	f.Topics[index][0]++
-	if f.MatchMessage(msg) {
+	if !f.MatchMessage(msg) {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed)
 	}
 	f.Topics[index][0]--
@@ -580,7 +573,8 @@ func TestMatchMessageAsym(t *testing.T) {
 
 	// topic mismatch
 	f.Topics[index][0]++
-	if f.MatchMessage(msg) {
+	if !f.MatchMessage(msg) {
+		// topic mismatch should have no affect, as topics are handled by topic matchers
 		t.Fatalf("failed MatchEnvelope(topic mismatch) with seed %d.", seed)
 	}
 	f.Topics[index][0]--
@@ -829,8 +823,9 @@ func TestVariableTopics(t *testing.T) {
 
 		f.Topics[i][lastTopicByte]++
 		match = f.MatchEnvelope(env)
-		if match {
+		if !match {
-			t.Fatalf("MatchEnvelope symmetric with seed %d, step %d: false positive.", seed, i)
+			// topic mismatch should have no affect, as topics are handled by topic matchers
+			t.Fatalf("MatchEnvelope symmetric with seed %d, step %d.", seed, i)
 		}
 	}
 }

whisper/whisperv6/peer.go

@@ -56,7 +56,7 @@ func newPeer(host *Whisper, remote *p2p.Peer, rw p2p.MsgReadWriter) *Peer {
 		powRequirement: 0.0,
 		known:          set.New(),
 		quit:           make(chan struct{}),
-		bloomFilter:    makeFullNodeBloom(),
+		bloomFilter:    MakeFullNodeBloom(),
 		fullNode:       true,
 	}
 }
@@ -120,7 +120,7 @@ func (peer *Peer) handshake() error {
 		err = s.Decode(&bloom)
 		if err == nil {
 			sz := len(bloom)
-			if sz != bloomFilterSize && sz != 0 {
+			if sz != BloomFilterSize && sz != 0 {
 				return fmt.Errorf("peer [%x] sent bad status message: wrong bloom filter size %d", peer.ID(), sz)
 			}
 			peer.setBloomFilter(bloom)
@@ -229,7 +229,7 @@ func (peer *Peer) notifyAboutBloomFilterChange(bloom []byte) error {
 func (peer *Peer) bloomMatch(env *Envelope) bool {
 	peer.bloomMu.Lock()
 	defer peer.bloomMu.Unlock()
-	return peer.fullNode || bloomFilterMatch(peer.bloomFilter, env.Bloom())
+	return peer.fullNode || BloomFilterMatch(peer.bloomFilter, env.Bloom())
 }
 
 func (peer *Peer) setBloomFilter(bloom []byte) {
@@ -238,13 +238,13 @@ func (peer *Peer) setBloomFilter(bloom []byte) {
 	peer.bloomFilter = bloom
 	peer.fullNode = isFullNode(bloom)
 	if peer.fullNode && peer.bloomFilter == nil {
-		peer.bloomFilter = makeFullNodeBloom()
+		peer.bloomFilter = MakeFullNodeBloom()
 	}
 }
 
-func makeFullNodeBloom() []byte {
+func MakeFullNodeBloom() []byte {
-	bloom := make([]byte, bloomFilterSize)
+	bloom := make([]byte, BloomFilterSize)
-	for i := 0; i < bloomFilterSize; i++ {
+	for i := 0; i < BloomFilterSize; i++ {
 		bloom[i] = 0xFF
 	}
 	return bloom

whisper/whisperv6/peer_test.go

@@ -23,6 +23,7 @@ import (
 	mrand "math/rand"
 	"net"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -71,7 +72,7 @@ var keys = []string{
 }
 
 type TestData struct {
-	started int
+	started int64
 	counter [NumNodes]int
 	mutex   sync.RWMutex
 }
@@ -151,7 +152,7 @@ func resetParams(t *testing.T) {
 }
 
 func initBloom(t *testing.T) {
-	masterBloomFilter = make([]byte, bloomFilterSize)
+	masterBloomFilter = make([]byte, BloomFilterSize)
 	_, err := mrand.Read(masterBloomFilter)
 	if err != nil {
 		t.Fatalf("rand failed: %s.", err)
@@ -163,7 +164,7 @@ func initBloom(t *testing.T) {
 		masterBloomFilter[i] = 0xFF
 	}
 
-	if !bloomFilterMatch(masterBloomFilter, msgBloom) {
+	if !BloomFilterMatch(masterBloomFilter, msgBloom) {
 		t.Fatalf("bloom mismatch on initBloom.")
 	}
 }
@@ -177,7 +178,7 @@ func initialize(t *testing.T) {
 
 	for i := 0; i < NumNodes; i++ {
 		var node TestNode
-		b := make([]byte, bloomFilterSize)
+		b := make([]byte, BloomFilterSize)
 		copy(b, masterBloomFilter)
 		node.shh = New(&DefaultConfig)
 		node.shh.SetMinimumPoW(masterPow)
@@ -240,9 +241,7 @@ func startServer(t *testing.T, s *p2p.Server) {
 		t.Fatal("failed to start the first server: ", err)
 	}
 
-	result.mutex.Lock()
+	atomic.AddInt64(&result.started, 1)
-	defer result.mutex.Unlock()
-	result.started++
 }
 
 func stopServers() {
@@ -472,7 +471,10 @@ func checkPowExchange(t *testing.T) {
 func checkBloomFilterExchangeOnce(t *testing.T, mustPass bool) bool {
 	for i, node := range nodes {
 		for peer := range node.shh.peers {
-			if !bytes.Equal(peer.bloomFilter, masterBloomFilter) {
+			peer.bloomMu.Lock()
+			equals := bytes.Equal(peer.bloomFilter, masterBloomFilter)
+			peer.bloomMu.Unlock()
+			if !equals {
 				if mustPass {
 					t.Fatalf("node %d: failed to exchange bloom filter requirement in round %d. \n%x expected \n%x got",
 						i, round, masterBloomFilter, peer.bloomFilter)
@@ -500,11 +502,13 @@ func checkBloomFilterExchange(t *testing.T) {
 
 func waitForServersToStart(t *testing.T) {
 	const iterations = 200
+	var started int64
 	for j := 0; j < iterations; j++ {
 		time.Sleep(50 * time.Millisecond)
-		if result.started == NumNodes {
+		started = atomic.LoadInt64(&result.started)
+		if started == NumNodes {
 			return
 		}
 	}
-	t.Fatalf("Failed to start all the servers, running: %d", result.started)
+	t.Fatalf("Failed to start all the servers, running: %d", started)
 }

whisper/whisperv6/whisper.go

@@ -232,11 +232,11 @@ func (whisper *Whisper) SetMaxMessageSize(size uint32) error {
 
 // SetBloomFilter sets the new bloom filter
 func (whisper *Whisper) SetBloomFilter(bloom []byte) error {
-	if len(bloom) != bloomFilterSize {
+	if len(bloom) != BloomFilterSize {
 		return fmt.Errorf("invalid bloom filter size: %d", len(bloom))
 	}
 
-	b := make([]byte, bloomFilterSize)
+	b := make([]byte, BloomFilterSize)
 	copy(b, bloom)
 
 	whisper.settings.Store(bloomFilterIdx, b)
@@ -558,14 +558,14 @@ func (whisper *Whisper) Subscribe(f *Filter) (string, error) {
 // updateBloomFilter recalculates the new value of bloom filter,
 // and informs the peers if necessary.
 func (whisper *Whisper) updateBloomFilter(f *Filter) {
-	aggregate := make([]byte, bloomFilterSize)
+	aggregate := make([]byte, BloomFilterSize)
 	for _, t := range f.Topics {
 		top := BytesToTopic(t)
 		b := TopicToBloom(top)
 		aggregate = addBloom(aggregate, b)
 	}
 
-	if !bloomFilterMatch(whisper.BloomFilter(), aggregate) {
+	if !BloomFilterMatch(whisper.BloomFilter(), aggregate) {
 		// existing bloom filter must be updated
 		aggregate = addBloom(whisper.BloomFilter(), aggregate)
 		whisper.SetBloomFilter(aggregate)
@@ -701,7 +701,7 @@ func (whisper *Whisper) runMessageLoop(p *Peer, rw p2p.MsgReadWriter) error {
 		case bloomFilterExCode:
 			var bloom []byte
 			err := packet.Decode(&bloom)
-			if err == nil && len(bloom) != bloomFilterSize {
+			if err == nil && len(bloom) != BloomFilterSize {
 				err = fmt.Errorf("wrong bloom filter size %d", len(bloom))
 			}
 
@@ -779,11 +779,11 @@ func (whisper *Whisper) add(envelope *Envelope, isP2P bool) (bool, error) {
 		}
 	}
 
-	if !bloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) {
+	if !BloomFilterMatch(whisper.BloomFilter(), envelope.Bloom()) {
 		// maybe the value was recently changed, and the peers did not adjust yet.
 		// in this case the previous value is retrieved by BloomFilterTolerance()
 		// for a short period of peer synchronization.
-		if !bloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) {
+		if !BloomFilterMatch(whisper.BloomFilterTolerance(), envelope.Bloom()) {
 			return false, fmt.Errorf("envelope does not match bloom filter, hash=[%v], bloom: \n%x \n%x \n%x",
 				envelope.Hash().Hex(), whisper.BloomFilter(), envelope.Bloom(), envelope.Topic)
 		}
@@ -928,24 +928,6 @@ func (whisper *Whisper) Envelopes() []*Envelope {
 	return all
 }
 
-// Messages iterates through all currently floating envelopes
-// and retrieves all the messages, that this filter could decrypt.
-func (whisper *Whisper) Messages(id string) []*ReceivedMessage {
-	result := make([]*ReceivedMessage, 0)
-	whisper.poolMu.RLock()
-	defer whisper.poolMu.RUnlock()
-
-	if filter := whisper.filters.Get(id); filter != nil {
-		for _, env := range whisper.envelopes {
-			msg := filter.processEnvelope(env)
-			if msg != nil {
-				result = append(result, msg)
-			}
-		}
-	}
-	return result
-}
-
 // isEnvelopeCached checks if envelope with specific hash has already been received and cached.
 func (whisper *Whisper) isEnvelopeCached(hash common.Hash) bool {
 	whisper.poolMu.Lock()
@@ -1043,12 +1025,12 @@ func isFullNode(bloom []byte) bool {
 	return true
 }
 
-func bloomFilterMatch(filter, sample []byte) bool {
+func BloomFilterMatch(filter, sample []byte) bool {
 	if filter == nil {
 		return true
 	}
 
-	for i := 0; i < bloomFilterSize; i++ {
+	for i := 0; i < BloomFilterSize; i++ {
 		f := filter[i]
 		s := sample[i]
 		if (f | s) != f {
@@ -1060,8 +1042,8 @@ func bloomFilterMatch(filter, sample []byte) bool {
 }
 
 func addBloom(a, b []byte) []byte {
-	c := make([]byte, bloomFilterSize)
+	c := make([]byte, BloomFilterSize)
-	for i := 0; i < bloomFilterSize; i++ {
+	for i := 0; i < BloomFilterSize; i++ {
 		c[i] = a[i] | b[i]
 	}
 	return c

whisper/whisperv6/whisper_test.go

@@ -75,10 +75,6 @@ func TestWhisperBasic(t *testing.T) {
 	if len(mail) != 0 {
 		t.Fatalf("failed w.Envelopes().")
 	}
-	m := w.Messages("non-existent")
-	if len(m) != 0 {
-		t.Fatalf("failed w.Messages.")
-	}
 
 	derived := pbkdf2.Key([]byte(peerID), nil, 65356, aesKeyLength, sha256.New)
 	if !validateDataIntegrity(derived, aesKeyLength) {
@@ -593,7 +589,7 @@ func TestCustomization(t *testing.T) {
 	}
 
 	// check w.messages()
-	id, err := w.Subscribe(f)
+	_, err = w.Subscribe(f)
 	if err != nil {
 		t.Fatalf("failed subscribe with seed %d: %s.", seed, err)
 	}
@@ -602,11 +598,6 @@ func TestCustomization(t *testing.T) {
 	if len(mail) > 0 {
 		t.Fatalf("received premature mail")
 	}
-
-	mail = w.Messages(id)
-	if len(mail) != 2 {
-		t.Fatalf("failed to get whisper messages")
-	}
 }
 
 func TestSymmetricSendCycle(t *testing.T) {
@@ -835,11 +826,11 @@ func TestSymmetricSendKeyMismatch(t *testing.T) {
 func TestBloom(t *testing.T) {
 	topic := TopicType{0, 0, 255, 6}
 	b := TopicToBloom(topic)
-	x := make([]byte, bloomFilterSize)
+	x := make([]byte, BloomFilterSize)
 	x[0] = byte(1)
 	x[32] = byte(1)
-	x[bloomFilterSize-1] = byte(128)
+	x[BloomFilterSize-1] = byte(128)
-	if !bloomFilterMatch(x, b) || !bloomFilterMatch(b, x) {
+	if !BloomFilterMatch(x, b) || !BloomFilterMatch(b, x) {
 		t.Fatalf("bloom filter does not match the mask")
 	}
 
@@ -851,11 +842,11 @@ func TestBloom(t *testing.T) {
 	if err != nil {
 		t.Fatalf("math rand error")
 	}
-	if !bloomFilterMatch(b, b) {
+	if !BloomFilterMatch(b, b) {
 		t.Fatalf("bloom filter does not match self")
 	}
 	x = addBloom(x, b)
-	if !bloomFilterMatch(x, b) {
+	if !BloomFilterMatch(x, b) {
 		t.Fatalf("bloom filter does not match combined bloom")
 	}
 	if !isFullNode(nil) {
@@ -865,16 +856,16 @@ func TestBloom(t *testing.T) {
 	if isFullNode(x) {
 		t.Fatalf("isFullNode false positive")
 	}
-	for i := 0; i < bloomFilterSize; i++ {
+	for i := 0; i < BloomFilterSize; i++ {
 		b[i] = byte(255)
 	}
 	if !isFullNode(b) {
 		t.Fatalf("isFullNode false negative")
 	}
-	if bloomFilterMatch(x, b) {
+	if BloomFilterMatch(x, b) {
 		t.Fatalf("bloomFilterMatch false positive")
 	}
-	if !bloomFilterMatch(b, x) {
+	if !BloomFilterMatch(b, x) {
 		t.Fatalf("bloomFilterMatch false negative")
 	}
 
@@ -888,7 +879,7 @@ func TestBloom(t *testing.T) {
 		t.Fatalf("failed to set bloom filter: %s", err)
 	}
 	f = w.BloomFilter()
-	if !bloomFilterMatch(f, x) || !bloomFilterMatch(x, f) {
+	if !BloomFilterMatch(f, x) || !BloomFilterMatch(x, f) {
 		t.Fatalf("retireved wrong bloom filter")
 	}
 }