swarm/storage: resolved merge conflicts

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+*.sol linguist-language=Solidity

.travis.yml

@@ -12,7 +12,7 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # These are the latest Go versions.
     - os: linux
@@ -24,7 +24,7 @@ matrix:
         - sudo chmod 666 /dev/fuse
         - sudo chown root:$USER /etc/fuse.conf
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     - os: osx
       go: "1.10"
@@ -34,7 +34,7 @@ matrix:
         - brew install caskroom/cask/brew-cask
         - brew cask install osxfuse
         - go run build/ci.go install
-        - go run build/ci.go test -coverage
+        - go run build/ci.go test -coverage $TEST_PACKAGES
 
     # This builder only tests code linters on latest version of Go
     - os: linux
@@ -47,14 +47,12 @@ matrix:
       script:
         - go run build/ci.go lint
 
-    # This builder does the Ubuntu PPA and Linux Azure uploads
+    # This builder does the Ubuntu PPA upload
     - os: linux
       dist: trusty
-      sudo: required
       go: "1.10"
       env:
         - ubuntu-ppa
-        - azure-linux
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -63,11 +61,25 @@ matrix:
             - devscripts
             - debhelper
             - dput
-            - gcc-multilib
             - fakeroot
       script:
-        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>" -upload ppa:ethereum/ethereum
+
+    # This builder does the Linux Azure uploads
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: "1.10"
+      env:
+        - azure-linux
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      addons:
+        apt:
+          packages:
+            - gcc-multilib
+      script:
+        # Build for the primary platforms that Trusty can manage
         - go run build/ci.go install
         - go run build/ci.go archive -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
         - go run build/ci.go install -arch 386
@@ -181,7 +193,6 @@ matrix:
     # This builder does the Azure archive purges to avoid accumulating junk
     - os: linux
       dist: trusty
-      sudo: required
       go: "1.10"
       env:
         - azure-purge

Dockerfile

@@ -12,5 +12,11 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/geth /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp
 ENTRYPOINT ["geth"]

Dockerfile.alltools

@@ -12,4 +12,10 @@ FROM alpine:latest
 RUN apk add --no-cache ca-certificates
 COPY --from=builder /go-ethereum/build/bin/* /usr/local/bin/
 
+RUN addgroup -g 1000 geth && \
+    adduser -h /root -D -u 1000 -G geth geth && \
+    chown geth:geth /root
+
+USER geth
+
 EXPOSE 8545 8546 30303 30303/udp 30304/udp

README.md

@@ -142,7 +142,7 @@ Do not forget `--rpcaddr 0.0.0.0`, if you want to access RPC from other containe
 ### Programatically interfacing Geth nodes
 
 As a developer, sooner rather than later you'll want to start interacting with Geth and the Ethereum
-network via your own programs and not manually through the console. To aid this, Geth has built in
+network via your own programs and not manually through the console. To aid this, Geth has built-in
 support for a JSON-RPC based APIs ([standard APIs](https://github.com/ethereum/wiki/wiki/JSON-RPC) and
 [Geth specific APIs](https://github.com/ethereum/go-ethereum/wiki/Management-APIs)). These can be
 exposed via HTTP, WebSockets and IPC (unix sockets on unix based platforms, and named pipes on Windows).

accounts/abi/numbers.go

@@ -25,23 +25,23 @@ import (
 )
 
 var (
-	big_t      = reflect.TypeOf(&big.Int{})
+	bigT      = reflect.TypeOf(&big.Int{})
-	derefbig_t = reflect.TypeOf(big.Int{})
+	derefbigT = reflect.TypeOf(big.Int{})
-	uint8_t    = reflect.TypeOf(uint8(0))
+	uint8T    = reflect.TypeOf(uint8(0))
-	uint16_t   = reflect.TypeOf(uint16(0))
+	uint16T   = reflect.TypeOf(uint16(0))
-	uint32_t   = reflect.TypeOf(uint32(0))
+	uint32T   = reflect.TypeOf(uint32(0))
-	uint64_t   = reflect.TypeOf(uint64(0))
+	uint64T   = reflect.TypeOf(uint64(0))
-	int_t      = reflect.TypeOf(int(0))
+	intT      = reflect.TypeOf(int(0))
-	int8_t     = reflect.TypeOf(int8(0))
+	int8T     = reflect.TypeOf(int8(0))
-	int16_t    = reflect.TypeOf(int16(0))
+	int16T    = reflect.TypeOf(int16(0))
-	int32_t    = reflect.TypeOf(int32(0))
+	int32T    = reflect.TypeOf(int32(0))
-	int64_t    = reflect.TypeOf(int64(0))
+	int64T    = reflect.TypeOf(int64(0))
-	address_t  = reflect.TypeOf(common.Address{})
+	addressT  = reflect.TypeOf(common.Address{})
-	int_ts     = reflect.TypeOf([]int(nil))
+	intTS     = reflect.TypeOf([]int(nil))
-	int8_ts    = reflect.TypeOf([]int8(nil))
+	int8TS    = reflect.TypeOf([]int8(nil))
-	int16_ts   = reflect.TypeOf([]int16(nil))
+	int16TS   = reflect.TypeOf([]int16(nil))
-	int32_ts   = reflect.TypeOf([]int32(nil))
+	int32TS   = reflect.TypeOf([]int32(nil))
-	int64_ts   = reflect.TypeOf([]int64(nil))
+	int64TS   = reflect.TypeOf([]int64(nil))
 )
 
 // U256 converts a big Int into a 256bit EVM number.
@@ -52,7 +52,7 @@ func U256(n *big.Int) []byte {
 // checks whether the given reflect value is signed. This also works for slices with a number type
 func isSigned(v reflect.Value) bool {
 	switch v.Type() {
-	case int_ts, int8_ts, int16_ts, int32_ts, int64_ts, int_t, int8_t, int16_t, int32_t, int64_t:
+	case intTS, int8TS, int16TS, int32TS, int64TS, intT, int8T, int16T, int32T, int64T:
 		return true
 	}
 	return false

accounts/abi/reflect.go

@@ -24,7 +24,7 @@ import (
 // indirect recursively dereferences the value until it either gets the value
 // or finds a big.Int
 func indirect(v reflect.Value) reflect.Value {
-	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbig_t {
+	if v.Kind() == reflect.Ptr && v.Elem().Type() != derefbigT {
 		return indirect(v.Elem())
 	}
 	return v
@@ -36,26 +36,26 @@ func reflectIntKindAndType(unsigned bool, size int) (reflect.Kind, reflect.Type)
 	switch size {
 	case 8:
 		if unsigned {
-			return reflect.Uint8, uint8_t
+			return reflect.Uint8, uint8T
 		}
-		return reflect.Int8, int8_t
+		return reflect.Int8, int8T
 	case 16:
 		if unsigned {
-			return reflect.Uint16, uint16_t
+			return reflect.Uint16, uint16T
 		}
-		return reflect.Int16, int16_t
+		return reflect.Int16, int16T
 	case 32:
 		if unsigned {
-			return reflect.Uint32, uint32_t
+			return reflect.Uint32, uint32T
 		}
-		return reflect.Int32, int32_t
+		return reflect.Int32, int32T
 	case 64:
 		if unsigned {
-			return reflect.Uint64, uint64_t
+			return reflect.Uint64, uint64T
 		}
-		return reflect.Int64, int64_t
+		return reflect.Int64, int64T
 	}
-	return reflect.Ptr, big_t
+	return reflect.Ptr, bigT
 }
 
 // mustArrayToBytesSlice creates a new byte slice with the exact same size as value

accounts/abi/type.go

@@ -135,7 +135,7 @@ func NewType(t string) (typ Type, err error) {
 		typ.Type = reflect.TypeOf(bool(false))
 	case "address":
 		typ.Kind = reflect.Array
-		typ.Type = address_t
+		typ.Type = addressT
 		typ.Size = 20
 		typ.T = AddressTy
 	case "string":

accounts/abi/type_test.go

@@ -46,36 +46,36 @@ func TestTypeRegexp(t *testing.T) {
 		{"bool[2][2][2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][2]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[2]"}, stringKind: "bool[2][2]"}, stringKind: "bool[2][2][2]"}},
 		{"bool[][][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][]"}, stringKind: "bool[][][]"}},
 		{"bool[][2][]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][2][]bool{}), Elem: &Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]bool{}), Elem: &Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]bool{}), Elem: &Type{Kind: reflect.Bool, T: BoolTy, Type: reflect.TypeOf(bool(false)), stringKind: "bool"}, stringKind: "bool[]"}, stringKind: "bool[][2]"}, stringKind: "bool[][2][]"}},
-		{"int8", Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}},
+		{"int8", Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}},
-		{"int16", Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}},
+		{"int16", Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}},
-		{"int32", Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}},
+		{"int32", Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}},
-		{"int64", Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}},
+		{"int64", Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}},
-		{"int256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}},
+		{"int256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}},
-		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
+		{"int8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[]"}},
-		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8_t, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
+		{"int8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int8{}), Elem: &Type{Kind: reflect.Int8, Type: int8T, Size: 8, T: IntTy, stringKind: "int8"}, stringKind: "int8[2]"}},
-		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
+		{"int16[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[]"}},
-		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16_t, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
+		{"int16[2]", Type{Size: 2, Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]int16{}), Elem: &Type{Kind: reflect.Int16, Type: int16T, Size: 16, T: IntTy, stringKind: "int16"}, stringKind: "int16[2]"}},
-		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
+		{"int32[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[]"}},
-		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32_t, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
+		{"int32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int32{}), Elem: &Type{Kind: reflect.Int32, Type: int32T, Size: 32, T: IntTy, stringKind: "int32"}, stringKind: "int32[2]"}},
-		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
+		{"int64[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[]"}},
-		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64_t, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
+		{"int64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]int64{}), Elem: &Type{Kind: reflect.Int64, Type: int64T, Size: 64, T: IntTy, stringKind: "int64"}, stringKind: "int64[2]"}},
-		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
+		{"int256[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[]"}},
-		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
+		{"int256[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: IntTy, stringKind: "int256"}, stringKind: "int256[2]"}},
-		{"uint8", Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}},
+		{"uint8", Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}},
-		{"uint16", Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}},
+		{"uint16", Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}},
-		{"uint32", Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}},
+		{"uint32", Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}},
-		{"uint64", Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}},
+		{"uint64", Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}},
-		{"uint256", Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}},
+		{"uint256", Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}},
-		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
+		{"uint8[]", Type{Kind: reflect.Slice, T: SliceTy, Type: reflect.TypeOf([]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[]"}},
-		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8_t, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
+		{"uint8[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint8{}), Elem: &Type{Kind: reflect.Uint8, Type: uint8T, Size: 8, T: UintTy, stringKind: "uint8"}, stringKind: "uint8[2]"}},
-		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
+		{"uint16[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[]"}},
-		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16_t, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
+		{"uint16[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint16{}), Elem: &Type{Kind: reflect.Uint16, Type: uint16T, Size: 16, T: UintTy, stringKind: "uint16"}, stringKind: "uint16[2]"}},
-		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
+		{"uint32[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[]"}},
-		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32_t, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
+		{"uint32[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint32{}), Elem: &Type{Kind: reflect.Uint32, Type: uint32T, Size: 32, T: UintTy, stringKind: "uint32"}, stringKind: "uint32[2]"}},
-		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
+		{"uint64[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[]"}},
-		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64_t, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
+		{"uint64[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]uint64{}), Elem: &Type{Kind: reflect.Uint64, Type: uint64T, Size: 64, T: UintTy, stringKind: "uint64"}, stringKind: "uint64[2]"}},
-		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
+		{"uint256[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]*big.Int{}), Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[]"}},
-		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: big_t, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
+		{"uint256[2]", Type{Kind: reflect.Array, T: ArrayTy, Type: reflect.TypeOf([2]*big.Int{}), Size: 2, Elem: &Type{Kind: reflect.Ptr, Type: bigT, Size: 256, T: UintTy, stringKind: "uint256"}, stringKind: "uint256[2]"}},
 		{"bytes32", Type{Kind: reflect.Array, T: FixedBytesTy, Size: 32, Type: reflect.TypeOf([32]byte{}), stringKind: "bytes32"}},
 		{"bytes[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([][]byte{}), Elem: &Type{Kind: reflect.Slice, Type: reflect.TypeOf([]byte{}), T: BytesTy, stringKind: "bytes"}, stringKind: "bytes[]"}},
 		{"bytes[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2][]byte{}), Elem: &Type{T: BytesTy, Type: reflect.TypeOf([]byte{}), Kind: reflect.Slice, stringKind: "bytes"}, stringKind: "bytes[2]"}},
@@ -84,9 +84,9 @@ func TestTypeRegexp(t *testing.T) {
 		{"string", Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}},
 		{"string[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]string{}), Elem: &Type{Kind: reflect.String, Type: reflect.TypeOf(""), T: StringTy, stringKind: "string"}, stringKind: "string[]"}},
 		{"string[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]string{}), Elem: &Type{Kind: reflect.String, T: StringTy, Type: reflect.TypeOf(""), stringKind: "string"}, stringKind: "string[2]"}},
-		{"address", Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}},
+		{"address", Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}},
-		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
+		{"address[]", Type{T: SliceTy, Kind: reflect.Slice, Type: reflect.TypeOf([]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[]"}},
-		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: address_t, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
+		{"address[2]", Type{Kind: reflect.Array, T: ArrayTy, Size: 2, Type: reflect.TypeOf([2]common.Address{}), Elem: &Type{Kind: reflect.Array, Type: addressT, Size: 20, T: AddressTy, stringKind: "address"}, stringKind: "address[2]"}},
 		// TODO when fixed types are implemented properly
 		// {"fixed", Type{}},
 		// {"fixed128x128", Type{}},
@@ -252,6 +252,9 @@ func TestTypeCheck(t *testing.T) {
 		{"bytes20", common.Address{}, ""},
 		{"address", [20]byte{}, ""},
 		{"address", common.Address{}, ""},
+		{"bytes32[]]", "", "invalid arg type in abi"},
+		{"invalidType", "", "unsupported arg type: invalidType"},
+		{"invalidSlice[]", "", "unsupported arg type: invalidSlice"},
 	} {
 		typ, err := NewType(test.typ)
 		if err != nil && len(test.err) == 0 {

accounts/abi/unpack_test.go

@@ -56,6 +56,23 @@ var unpackTests = []unpackTest{
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",
 		want: true,
 	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000000",
+		want: false,
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000001000000000001",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
+	{
+		def:  `[{ "type": "bool" }]`,
+		enc:  "0000000000000000000000000000000000000000000000000000000000000003",
+		want: false,
+		err:  "abi: improperly encoded boolean value",
+	},
 	{
 		def:  `[{"type": "uint32"}]`,
 		enc:  "0000000000000000000000000000000000000000000000000000000000000001",

bmt/bmt.go

@@ -75,7 +75,7 @@ type Hasher struct {
 	blocksize   int         // segment size (size of hash) also for hash.Hash
 	count       int         // segment count
 	size        int         // for hash.Hash same as hashsize
-	cur         int         // cursor position for righmost currently open chunk
+	cur         int         // cursor position for rightmost currently open chunk
 	segment     []byte      // the rightmost open segment (not complete)
 	depth       int         // index of last level
 	result      chan []byte // result channel
@@ -149,7 +149,7 @@ func NewTreePool(hasher BaseHasher, segmentCount, capacity int) *TreePool {
 	}
 }
 
-// Drain drains the pool uptil it has no more than n resources
+// Drain drains the pool until it has no more than n resources
 func (self *TreePool) Drain(n int) {
 	self.lock.Lock()
 	defer self.lock.Unlock()
@@ -412,11 +412,10 @@ func (self *Hasher) Reset() {
 
 // ResetWithLength needs to be called before writing to the hasher
 // the argument is supposed to be the byte slice binary representation of
-// the legth of the data subsumed under the hash
+// the length of the data subsumed under the hash
 func (self *Hasher) ResetWithLength(l []byte) {
 	self.Reset()
 	self.blockLength = l
-
 }
 
 // Release gives back the Tree to the pool whereby it unlocks
@@ -531,7 +530,7 @@ func (self *Hasher) finalise(n *Node, i int) (d int) {
 	for {
 		// when the final segment's path is going via left segments
 		// the incoming data is pushed to the parent upon pulling the left
-		// we do not need toogle the state since this condition is
+		// we do not need toggle the state since this condition is
 		// detectable
 		n.unbalanced = isLeft
 		n.right = nil

build/ci.go

@@ -766,7 +766,7 @@ func doAndroidArchive(cmdline []string) {
 		if meta.Develop {
 			repo = *deploy + "/content/repositories/snapshots"
 		}
-		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file",
+		build.MustRunCommand("mvn", "gpg:sign-and-deploy-file", "-e", "-X",
 			"-settings=build/mvn.settings", "-Durl="+repo, "-DrepositoryId=ossrh",
 			"-DpomFile="+meta.Package+".pom", "-Dfile="+meta.Package+".aar")
 	}

cmd/evm/runner.go

@@ -76,6 +76,7 @@ func runCmd(ctx *cli.Context) error {
 	logconfig := &vm.LogConfig{
 		DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name),
 		DisableStack:  ctx.GlobalBool(DisableStackFlag.Name),
+		Debug:         ctx.GlobalBool(DebugFlag.Name),
 	}
 
 	var (
@@ -83,8 +84,8 @@ func runCmd(ctx *cli.Context) error {
 		debugLogger *vm.StructLogger
 		statedb     *state.StateDB
 		chainConfig *params.ChainConfig
-		sender      = common.StringToAddress("sender")
+		sender      = common.BytesToAddress([]byte("sender"))
-		receiver    = common.StringToAddress("receiver")
+		receiver    = common.BytesToAddress([]byte("receiver"))
 	)
 	if ctx.GlobalBool(MachineFlag.Name) {
 		tracer = NewJSONLogger(logconfig, os.Stdout)
@@ -234,9 +235,7 @@ Gas used:           %d
 
 `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas)
 	}
-	if tracer != nil {
+	if tracer == nil {
-		tracer.CaptureEnd(ret, initialGas-leftOverGas, execTime, err)
-	} else {
 		fmt.Printf("0x%x\n", ret)
 		if err != nil {
 			fmt.Printf(" error: %v\n", err)

cmd/geth/bugcmd.go

@@ -49,15 +49,17 @@ func reportBug(ctx *cli.Context) error {
 	// execute template and write contents to buff
 	var buff bytes.Buffer
 
-	fmt.Fprintln(&buff, header)
+	fmt.Fprintln(&buff, "#### System information")
+	fmt.Fprintln(&buff)
 	fmt.Fprintln(&buff, "Version:", params.Version)
 	fmt.Fprintln(&buff, "Go Version:", runtime.Version())
 	fmt.Fprintln(&buff, "OS:", runtime.GOOS)
 	printOSDetails(&buff)
+	fmt.Fprintln(&buff, header)
 
 	// open a new GH issue
 	if !browser.Open(issueUrl + "?body=" + url.QueryEscape(buff.String())) {
-		fmt.Printf("Please file a new issue at %s using this template:\n%s", issueUrl, buff.String())
+		fmt.Printf("Please file a new issue at %s using this template:\n\n%s", issueUrl, buff.String())
 	}
 	return nil
 }
@@ -97,13 +99,15 @@ func printCmdOut(w io.Writer, prefix, path string, args ...string) {
 	fmt.Fprintf(w, "%s%s\n", prefix, bytes.TrimSpace(out))
 }
 
-const header = `Please answer these questions before submitting your issue. Thanks!
+const header = `
+#### Expected behaviour
 
-#### What did you do?
 
-#### What did you expect to see?
+#### Actual behaviour
 
-#### What did you see instead?
 
-#### System details
+#### Steps to reproduce the behaviour
+
+
+#### Backtrace
 `

cmd/geth/main.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/cmd/utils"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -46,8 +45,6 @@ const (
 var (
 	// Git SHA1 commit hash of the release (set via linker flags)
 	gitCommit = ""
-	// Ethereum address of the Geth release oracle.
-	relOracle = common.HexToAddress("0xfa7b9770ca4cb04296cac84f37736d4041251cdf")
 	// The app that holds all commands and flags.
 	app = utils.NewApp(gitCommit, "the go-ethereum command line interface")
 	// flags that configure the node

cmd/puppeth/module_node.go

@@ -40,11 +40,11 @@ ADD genesis.json /genesis.json
 	ADD signer.pass /signer.pass
 {{end}}
 RUN \
-  echo 'geth --cache 512 init /genesis.json' > geth.sh && \{{if .Unlock}}
+  echo 'geth --cache 512 init /genesis.json' > /root/geth.sh && \{{if .Unlock}}
-	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> geth.sh && \{{end}}
+	echo 'mkdir -p /root/.ethereum/keystore/ && cp /signer.json /root/.ethereum/keystore/' >> /root/geth.sh && \{{end}}
-	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> geth.sh
+	echo $'geth --networkid {{.NetworkID}} --cache 512 --port {{.Port}} --maxpeers {{.Peers}} {{.LightFlag}} --ethstats \'{{.Ethstats}}\' {{if .Bootnodes}}--bootnodes {{.Bootnodes}}{{end}} {{if .Etherbase}}--etherbase {{.Etherbase}} --mine --minerthreads 1{{end}} {{if .Unlock}}--unlock 0 --password /signer.pass --mine{{end}} --targetgaslimit {{.GasTarget}} --gasprice {{.GasPrice}}' >> /root/geth.sh
 
-ENTRYPOINT ["/bin/sh", "geth.sh"]
+ENTRYPOINT ["/bin/sh", "/root/geth.sh"]
 `
 
 // nodeComposefile is the docker-compose.yml file required to deploy and maintain

cmd/swarm/main.go

@@ -141,6 +141,10 @@ var (
 		Name:  "mime",
 		Usage: "force mime type",
 	}
+	SwarmEncryptedFlag = cli.BoolFlag{
+		Name:  "encrypted",
+		Usage: "use encrypted upload",
+	}
 	SwarmPssEnabledFlag = cli.BoolFlag{
 		Name:  "pss",
 		Usage: "Enable pss (message passing over swarm)",
@@ -222,6 +226,7 @@ The output of this command is supposed to be machine-readable.
 			Name:      "up",
 			Usage:     "upload a file or directory to swarm using the HTTP API",
 			ArgsUsage: " <file>",
+			Flags:     []cli.Flag{SwarmEncryptedFlag},
 			Description: `
 "upload a file or directory to swarm using the HTTP API and prints the root hash",
 `,

cmd/swarm/manifest.go

@@ -131,13 +131,13 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.DownloadManifest(mhash)
+	mroot, isEncrypted, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
 
 	//TODO: check if the "hash" to add is valid and present in swarm
-	_, err = client.DownloadManifest(hash)
+	_, _, err = client.DownloadManifest(hash)
 	if err != nil {
 		utils.Fatalf("Hash to add is not present: %v", err)
 	}
@@ -180,7 +180,7 @@ func addEntryToManifest(ctx *cli.Context, mhash, path, hash, ctype string) strin
 		mroot.Entries = append(mroot.Entries, newEntry)
 	}
 
-	newManifestHash, err := client.UploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -197,7 +197,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.DownloadManifest(mhash)
+	mroot, isEncrypted, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -257,7 +257,7 @@ func updateEntryInManifest(ctx *cli.Context, mhash, path, hash, ctype string) st
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.UploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}
@@ -273,7 +273,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		longestPathEntry = api.ManifestEntry{}
 	)
 
-	mroot, err := client.DownloadManifest(mhash)
+	mroot, isEncrypted, err := client.DownloadManifest(mhash)
 	if err != nil {
 		utils.Fatalf("Manifest download failed: %v", err)
 	}
@@ -323,7 +323,7 @@ func removeEntryFromManifest(ctx *cli.Context, mhash, path string) string {
 		mroot = newMRoot
 	}
 
-	newManifestHash, err := client.UploadManifest(mroot)
+	newManifestHash, err := client.UploadManifest(mroot, isEncrypted)
 	if err != nil {
 		utils.Fatalf("Manifest upload failed: %v", err)
 	}

cmd/swarm/upload.go

@@ -46,6 +46,7 @@ func upload(ctx *cli.Context) {
 		fromStdin    = ctx.GlobalBool(SwarmUpFromStdinFlag.Name)
 		mimeType     = ctx.GlobalString(SwarmUploadMimeType.Name)
 		client       = swarm.NewClient(bzzapi)
+		toEncrypt    = ctx.Bool(SwarmEncryptedFlag.Name)
 		file         string
 	)
 
@@ -76,7 +77,7 @@ func upload(ctx *cli.Context) {
 			utils.Fatalf("Error opening file: %s", err)
 		}
 		defer f.Close()
-		hash, err := client.UploadRaw(f, f.Size)
+		hash, err := client.UploadRaw(f, f.Size, toEncrypt)
 		if err != nil {
 			utils.Fatalf("Upload failed: %s", err)
 		}
@@ -97,7 +98,7 @@ func upload(ctx *cli.Context) {
 			if !recursive {
 				return "", errors.New("Argument is a directory and recursive upload is disabled")
 			}
-			return client.UploadDirectory(file, defaultPath, "")
+			return client.UploadDirectory(file, defaultPath, "", toEncrypt)
 		}
 	} else {
 		doUpload = func() (string, error) {
@@ -110,7 +111,7 @@ func upload(ctx *cli.Context) {
 				mimeType = detectMimeType(file)
 			}
 			f.ContentType = mimeType
-			return client.Upload(f, "")
+			return client.Upload(f, "", toEncrypt)
 		}
 	}
 	hash, err := doUpload()

cmd/swarm/upload_test.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -29,6 +30,16 @@ import (
 // TestCLISwarmUp tests that running 'swarm up' makes the resulting file
 // available from all nodes via the HTTP API
 func TestCLISwarmUp(t *testing.T) {
+	testCLISwarmUp(false, t)
+}
+
+// TestCLISwarmUpEncrypted tests that running 'swarm encrypted-up' makes the resulting file
+// available from all nodes via the HTTP API
+func TestCLISwarmUpEncrypted(t *testing.T) {
+	testCLISwarmUp(true, t)
+}
+
+func testCLISwarmUp(toEncrypt bool, t *testing.T) {
 	log.Info("starting 3 node cluster")
 	cluster := newTestCluster(t, 3)
 	defer cluster.Shutdown()
@@ -48,10 +59,23 @@ func TestCLISwarmUp(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	hashRegexp := `[a-f\d]{64}`
+	flags := []string{
+		"--bzzapi", cluster.Nodes[0].URL,
+		"up",
+		tmp.Name()}
+	if toEncrypt {
+		hashRegexp = `[a-f\d]{128}`
+		flags = []string{
+			"--bzzapi", cluster.Nodes[0].URL,
+			"up",
+			"--encrypted",
+			tmp.Name()}
+	}
 	// upload the file with 'swarm up' and expect a hash
-	log.Info("uploading file with 'swarm up'")
+	log.Info(fmt.Sprintf("uploading file with 'swarm up'"))
-	up := runSwarm(t, "--bzzapi", cluster.Nodes[0].URL, "up", tmp.Name())
+	up := runSwarm(t, flags...)
-	_, matches := up.ExpectRegexp(`[a-f\d]{64}`)
+	_, matches := up.ExpectRegexp(hashRegexp)
 	up.ExpectExit()
 	hash := matches[0]
 	log.Info("file uploaded", "hash", hash)

common/types.go

@@ -45,7 +45,6 @@ func BytesToHash(b []byte) Hash {
 	h.SetBytes(b)
 	return h
 }
-func StringToHash(s string) Hash { return BytesToHash([]byte(s)) }
 func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) }
 func HexToHash(s string) Hash   { return BytesToHash(FromHex(s)) }
 
@@ -143,7 +142,6 @@ func BytesToAddress(b []byte) Address {
 	a.SetBytes(b)
 	return a
 }
-func StringToAddress(s string) Address { return BytesToAddress([]byte(s)) }
 func BigToAddress(b *big.Int) Address { return BytesToAddress(b.Bytes()) }
 func HexToAddress(s string) Address   { return BytesToAddress(FromHex(s)) }
 

compression/rle/read_write.go

@@ -1,101 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package rle implements the run-length encoding used for Ethereum data.
-package rle
-
-import (
-	"bytes"
-	"errors"
-
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-const (
-	token             byte = 0xfe
-	emptyShaToken          = 0xfd
-	emptyListShaToken      = 0xfe
-	tokenToken             = 0xff
-)
-
-var empty = crypto.Keccak256([]byte(""))
-var emptyList = crypto.Keccak256([]byte{0x80})
-
-func Decompress(dat []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-
-	for i := 0; i < len(dat); i++ {
-		if dat[i] == token {
-			if i+1 < len(dat) {
-				switch dat[i+1] {
-				case emptyShaToken:
-					buf.Write(empty)
-				case emptyListShaToken:
-					buf.Write(emptyList)
-				case tokenToken:
-					buf.WriteByte(token)
-				default:
-					buf.Write(make([]byte, int(dat[i+1]-2)))
-				}
-				i++
-			} else {
-				return nil, errors.New("error reading bytes. token encountered without proceeding bytes")
-			}
-		} else {
-			buf.WriteByte(dat[i])
-		}
-	}
-
-	return buf.Bytes(), nil
-}
-
-func compressChunk(dat []byte) (ret []byte, n int) {
-	switch {
-	case dat[0] == token:
-		return []byte{token, tokenToken}, 1
-	case len(dat) > 1 && dat[0] == 0x0 && dat[1] == 0x0:
-		j := 0
-		for j <= 254 && j < len(dat) {
-			if dat[j] != 0 {
-				break
-			}
-			j++
-		}
-		return []byte{token, byte(j + 2)}, j
-	case len(dat) >= 32:
-		if dat[0] == empty[0] && bytes.Equal(dat[:32], empty) {
-			return []byte{token, emptyShaToken}, 32
-		} else if dat[0] == emptyList[0] && bytes.Equal(dat[:32], emptyList) {
-			return []byte{token, emptyListShaToken}, 32
-		}
-		fallthrough
-	default:
-		return dat[:1], 1
-	}
-}
-
-func Compress(dat []byte) []byte {
-	buf := new(bytes.Buffer)
-
-	i := 0
-	for i < len(dat) {
-		b, n := compressChunk(dat[i:])
-		buf.Write(b)
-		i += n
-	}
-
-	return buf.Bytes()
-}

compression/rle/read_write_test.go

@@ -1,50 +0,0 @@
-// Copyright 2014 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package rle
-
-import (
-	"testing"
-
-	checker "gopkg.in/check.v1"
-)
-
-func Test(t *testing.T) { checker.TestingT(t) }
-
-type CompressionRleSuite struct{}
-
-var _ = checker.Suite(&CompressionRleSuite{})
-
-func (s *CompressionRleSuite) TestDecompressSimple(c *checker.C) {
-	exp := []byte{0xc5, 0xd2, 0x46, 0x1, 0x86, 0xf7, 0x23, 0x3c, 0x92, 0x7e, 0x7d, 0xb2, 0xdc, 0xc7, 0x3, 0xc0, 0xe5, 0x0, 0xb6, 0x53, 0xca, 0x82, 0x27, 0x3b, 0x7b, 0xfa, 0xd8, 0x4, 0x5d, 0x85, 0xa4, 0x70}
-	res, err := Decompress([]byte{token, 0xfd})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	exp = []byte{0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, 0x1, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21}
-	res, err = Decompress([]byte{token, 0xfe})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, exp)
-
-	res, err = Decompress([]byte{token, 0xff})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, []byte{token})
-
-	res, err = Decompress([]byte{token, 12})
-	c.Assert(err, checker.IsNil)
-	c.Assert(res, checker.DeepEquals, make([]byte, 10))
-
-}

core/blockchain_test.go

@@ -1338,3 +1338,114 @@ func TestLargeReorgTrieGC(t *testing.T) {
 		}
 	}
 }
+
+// Benchmarks large blocks with value transfers to non-existing accounts
+func benchmarkLargeNumberOfValueToNonexisting(b *testing.B, numTxs, numBlocks int, recipientFn func(uint64) common.Address, dataFn func(uint64) []byte) {
+	var (
+		signer          = types.HomesteadSigner{}
+		testBankKey, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+		bankFunds       = big.NewInt(100000000000000000)
+		gspec           = Genesis{
+			Config: params.TestChainConfig,
+			Alloc: GenesisAlloc{
+				testBankAddress: {Balance: bankFunds},
+				common.HexToAddress("0xc0de"): {
+					Code:    []byte{0x60, 0x01, 0x50},
+					Balance: big.NewInt(0),
+				}, // push 1, pop
+			},
+			GasLimit: 100e6, // 100 M
+		}
+	)
+	// Generate the original common chain segment and the two competing forks
+	engine := ethash.NewFaker()
+	db, _ := ethdb.NewMemDatabase()
+	genesis := gspec.MustCommit(db)
+
+	blockGenerator := func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{1})
+		for txi := 0; txi < numTxs; txi++ {
+			uniq := uint64(i*numTxs + txi)
+			recipient := recipientFn(uniq)
+			//recipient := common.BigToAddress(big.NewInt(0).SetUint64(1337 + uniq))
+			tx, err := types.SignTx(types.NewTransaction(uniq, recipient, big.NewInt(1), params.TxGas, big.NewInt(1), nil), signer, testBankKey)
+			if err != nil {
+				b.Error(err)
+			}
+			block.AddTx(tx)
+		}
+	}
+
+	shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, numBlocks, blockGenerator)
+	b.StopTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		// Import the shared chain and the original canonical one
+		diskdb, _ := ethdb.NewMemDatabase()
+		gspec.MustCommit(diskdb)
+
+		chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{})
+		if err != nil {
+			b.Fatalf("failed to create tester chain: %v", err)
+		}
+		b.StartTimer()
+		if _, err := chain.InsertChain(shared); err != nil {
+			b.Fatalf("failed to insert shared chain: %v", err)
+		}
+		b.StopTimer()
+		if got := chain.CurrentBlock().Transactions().Len(); got != numTxs*numBlocks {
+			b.Fatalf("Transactions were not included, expected %d, got %d", (numTxs * numBlocks), got)
+
+		}
+	}
+}
+func BenchmarkBlockChain_1x1000ValueTransferToNonexisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337 + nonce))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000ValueTransferToExisting(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(1337))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}
+func BenchmarkBlockChain_1x1000Executions(b *testing.B) {
+	var (
+		numTxs    = 1000
+		numBlocks = 1
+	)
+	b.StopTimer()
+	b.ResetTimer()
+
+	recipientFn := func(nonce uint64) common.Address {
+		return common.BigToAddress(big.NewInt(0).SetUint64(0xc0de))
+	}
+	dataFn := func(nonce uint64) []byte {
+		return nil
+	}
+
+	benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn)
+}

core/database_util_test.go

@@ -317,7 +317,7 @@ func TestLookupStorage(t *testing.T) {
 			if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
 				t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
 			}
-			if tx.String() != txn.String() {
+			if tx.Hash() != txn.Hash() {
 				t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
 			}
 		}

core/state/database.go

@@ -26,7 +26,7 @@ import (
 	lru "github.com/hashicorp/golang-lru"
 )
 
-// Trie cache generation limit after which to evic trie nodes from memory.
+// Trie cache generation limit after which to evict trie nodes from memory.
 var MaxTrieCacheGen = uint16(120)
 
 const (
@@ -151,9 +151,6 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 		return cached.(int), nil
 	}
 	code, err := db.ContractCode(addrHash, codeHash)
-	if err == nil {
-		db.codeSizeCache.Add(codeHash, len(code))
-	}
 	return len(code), err
 }
 

core/state/dump.go

@@ -53,7 +53,7 @@ func (self *StateDB) RawDump() Dump {
 			panic(err)
 		}
 
-		obj := newObject(nil, common.BytesToAddress(addr), data, nil)
+		obj := newObject(nil, common.BytesToAddress(addr), data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,

core/state/journal.go

@@ -22,11 +22,67 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+// journalEntry is a modification entry in the state change journal that can be
+// reverted on demand.
 type journalEntry interface {
-	undo(*StateDB)
+	// revert undoes the changes introduced by this journal entry.
+	revert(*StateDB)
+
+	// dirtied returns the Ethereum address modified by this journal entry.
+	dirtied() *common.Address
 }
 
-type journal []journalEntry
+// journal contains the list of state modifications applied since the last state
+// commit. These are tracked to be able to be reverted in case of an execution
+// exception or revertal request.
+type journal struct {
+	entries []journalEntry         // Current changes tracked by the journal
+	dirties map[common.Address]int // Dirty accounts and the number of changes
+}
+
+// newJournal create a new initialized journal.
+func newJournal() *journal {
+	return &journal{
+		dirties: make(map[common.Address]int),
+	}
+}
+
+// append inserts a new modification entry to the end of the change journal.
+func (j *journal) append(entry journalEntry) {
+	j.entries = append(j.entries, entry)
+	if addr := entry.dirtied(); addr != nil {
+		j.dirties[*addr]++
+	}
+}
+
+// revert undoes a batch of journalled modifications along with any reverted
+// dirty handling too.
+func (j *journal) revert(statedb *StateDB, snapshot int) {
+	for i := len(j.entries) - 1; i >= snapshot; i-- {
+		// Undo the changes made by the operation
+		j.entries[i].revert(statedb)
+
+		// Drop any dirty tracking induced by the change
+		if addr := j.entries[i].dirtied(); addr != nil {
+			if j.dirties[*addr]--; j.dirties[*addr] == 0 {
+				delete(j.dirties, *addr)
+			}
+		}
+	}
+	j.entries = j.entries[:snapshot]
+}
+
+// dirty explicitly sets an address to dirty, even if the change entries would
+// otherwise suggest it as clean. This method is an ugly hack to handle the RIPEMD
+// precompile consensus exception.
+func (j *journal) dirty(addr common.Address) {
+	j.dirties[addr]++
+}
+
+// length returns the current number of entries in the journal.
+func (j *journal) length() int {
+	return len(j.entries)
+}
 
 type (
 	// Changes to the account trie.
@@ -77,16 +133,24 @@ type (
 	}
 )
 
-func (ch createObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) revert(s *StateDB) {
 	delete(s.stateObjects, *ch.account)
 	delete(s.stateObjectsDirty, *ch.account)
 }
 
-func (ch resetObjectChange) undo(s *StateDB) {
+func (ch createObjectChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch resetObjectChange) revert(s *StateDB) {
 	s.setStateObject(ch.prev)
 }
 
-func (ch suicideChange) undo(s *StateDB) {
+func (ch resetObjectChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch suicideChange) revert(s *StateDB) {
 	obj := s.getStateObject(*ch.account)
 	if obj != nil {
 		obj.suicided = ch.prev
@@ -94,38 +158,60 @@ func (ch suicideChange) undo(s *StateDB) {
 	}
 }
 
+func (ch suicideChange) dirtied() *common.Address {
+	return ch.account
+}
+
 var ripemd = common.HexToAddress("0000000000000000000000000000000000000003")
 
-func (ch touchChange) undo(s *StateDB) {
+func (ch touchChange) revert(s *StateDB) {
-	if !ch.prev && *ch.account != ripemd {
-		s.getStateObject(*ch.account).touched = ch.prev
-		if !ch.prevDirty {
-			delete(s.stateObjectsDirty, *ch.account)
-		}
-	}
 }
 
-func (ch balanceChange) undo(s *StateDB) {
+func (ch touchChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch balanceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setBalance(ch.prev)
 }
 
-func (ch nonceChange) undo(s *StateDB) {
+func (ch balanceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch nonceChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setNonce(ch.prev)
 }
 
-func (ch codeChange) undo(s *StateDB) {
+func (ch nonceChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch codeChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setCode(common.BytesToHash(ch.prevhash), ch.prevcode)
 }
 
-func (ch storageChange) undo(s *StateDB) {
+func (ch codeChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch storageChange) revert(s *StateDB) {
 	s.getStateObject(*ch.account).setState(ch.key, ch.prevalue)
 }
 
-func (ch refundChange) undo(s *StateDB) {
+func (ch storageChange) dirtied() *common.Address {
+	return ch.account
+}
+
+func (ch refundChange) revert(s *StateDB) {
 	s.refund = ch.prev
 }
 
-func (ch addLogChange) undo(s *StateDB) {
+func (ch refundChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addLogChange) revert(s *StateDB) {
 	logs := s.logs[ch.txhash]
 	if len(logs) == 1 {
 		delete(s.logs, ch.txhash)
@@ -135,6 +221,14 @@ func (ch addLogChange) undo(s *StateDB) {
 	s.logSize--
 }
 
-func (ch addPreimageChange) undo(s *StateDB) {
+func (ch addLogChange) dirtied() *common.Address {
+	return nil
+}
+
+func (ch addPreimageChange) revert(s *StateDB) {
 	delete(s.preimages, ch.hash)
 }
+
+func (ch addPreimageChange) dirtied() *common.Address {
+	return nil
+}

core/state/state_object.go

@@ -85,9 +85,7 @@ type stateObject struct {
 	// during the "update" phase of the state transition.
 	dirtyCode bool // true if the code was updated
 	suicided  bool
-	touched   bool
 	deleted   bool
-	onDirty   func(addr common.Address) // Callback method to mark a state object newly dirty
 }
 
 // empty returns whether the account is considered empty.
@@ -105,7 +103,7 @@ type Account struct {
 }
 
 // newObject creates a state object.
-func newObject(db *StateDB, address common.Address, data Account, onDirty func(addr common.Address)) *stateObject {
+func newObject(db *StateDB, address common.Address, data Account) *stateObject {
 	if data.Balance == nil {
 		data.Balance = new(big.Int)
 	}
@@ -119,7 +117,6 @@ func newObject(db *StateDB, address common.Address, data Account, onDirty func(a
 		data:          data,
 		cachedStorage: make(Storage),
 		dirtyStorage:  make(Storage),
-		onDirty:       onDirty,
 	}
 }
 
@@ -137,23 +134,17 @@ func (self *stateObject) setError(err error) {
 
 func (self *stateObject) markSuicided() {
 	self.suicided = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (c *stateObject) touch() {
-	c.db.journal = append(c.db.journal, touchChange{
+	c.db.journal.append(touchChange{
 		account: &c.address,
-		prev:      c.touched,
-		prevDirty: c.onDirty == nil,
 	})
-	if c.onDirty != nil {
+	if c.address == ripemd {
-		c.onDirty(c.Address())
+		// Explicitly put it in the dirty-cache, which is otherwise generated from
-		c.onDirty = nil
+		// flattened journals.
+		c.db.journal.dirty(c.address)
 	}
-	c.touched = true
 }
 
 func (c *stateObject) getTrie(db Database) Trie {
@@ -195,7 +186,7 @@ func (self *stateObject) GetState(db Database, key common.Hash) common.Hash {
 
 // SetState updates a value in account storage.
 func (self *stateObject) SetState(db Database, key, value common.Hash) {
-	self.db.journal = append(self.db.journal, storageChange{
+	self.db.journal.append(storageChange{
 		account:  &self.address,
 		key:      key,
 		prevalue: self.GetState(db, key),
@@ -207,10 +198,6 @@ func (self *stateObject) setState(key, value common.Hash) {
 	self.cachedStorage[key] = value
 	self.dirtyStorage[key] = value
 
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // updateTrie writes cached storage modifications into the object's storage trie.
@@ -274,7 +261,7 @@ func (c *stateObject) SubBalance(amount *big.Int) {
 }
 
 func (self *stateObject) SetBalance(amount *big.Int) {
-	self.db.journal = append(self.db.journal, balanceChange{
+	self.db.journal.append(balanceChange{
 		account: &self.address,
 		prev:    new(big.Int).Set(self.data.Balance),
 	})
@@ -283,17 +270,13 @@ func (self *stateObject) SetBalance(amount *big.Int) {
 
 func (self *stateObject) setBalance(amount *big.Int) {
 	self.data.Balance = amount
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 // Return the gas back to the origin. Used by the Virtual machine or Closures
 func (c *stateObject) ReturnGas(gas *big.Int) {}
 
-func (self *stateObject) deepCopy(db *StateDB, onDirty func(addr common.Address)) *stateObject {
+func (self *stateObject) deepCopy(db *StateDB) *stateObject {
-	stateObject := newObject(db, self.address, self.data, onDirty)
+	stateObject := newObject(db, self.address, self.data)
 	if self.trie != nil {
 		stateObject.trie = db.db.CopyTrie(self.trie)
 	}
@@ -333,7 +316,7 @@ func (self *stateObject) Code(db Database) []byte {
 
 func (self *stateObject) SetCode(codeHash common.Hash, code []byte) {
 	prevcode := self.Code(self.db.db)
-	self.db.journal = append(self.db.journal, codeChange{
+	self.db.journal.append(codeChange{
 		account:  &self.address,
 		prevhash: self.CodeHash(),
 		prevcode: prevcode,
@@ -345,14 +328,10 @@ func (self *stateObject) setCode(codeHash common.Hash, code []byte) {
 	self.code = code
 	self.data.CodeHash = codeHash[:]
 	self.dirtyCode = true
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) SetNonce(nonce uint64) {
-	self.db.journal = append(self.db.journal, nonceChange{
+	self.db.journal.append(nonceChange{
 		account: &self.address,
 		prev:    self.data.Nonce,
 	})
@@ -361,10 +340,6 @@ func (self *stateObject) SetNonce(nonce uint64) {
 
 func (self *stateObject) setNonce(nonce uint64) {
 	self.data.Nonce = nonce
-	if self.onDirty != nil {
-		self.onDirty(self.Address())
-		self.onDirty = nil
-	}
 }
 
 func (self *stateObject) CodeHash() []byte {

core/state/statedb.go

@@ -76,7 +76,7 @@ type StateDB struct {
 
 	// Journal of state modifications. This is the backbone of
 	// Snapshot and RevertToSnapshot.
-	journal        journal
+	journal        *journal
 	validRevisions []revision
 	nextRevisionId int
 
@@ -96,6 +96,7 @@ func New(root common.Hash, db Database) (*StateDB, error) {
 		stateObjectsDirty: make(map[common.Address]struct{}),
 		logs:              make(map[common.Hash][]*types.Log),
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}, nil
 }
 
@@ -131,7 +132,7 @@ func (self *StateDB) Reset(root common.Hash) error {
 }
 
 func (self *StateDB) AddLog(log *types.Log) {
-	self.journal = append(self.journal, addLogChange{txhash: self.thash})
+	self.journal.append(addLogChange{txhash: self.thash})
 
 	log.TxHash = self.thash
 	log.BlockHash = self.bhash
@@ -156,7 +157,7 @@ func (self *StateDB) Logs() []*types.Log {
 // AddPreimage records a SHA3 preimage seen by the VM.
 func (self *StateDB) AddPreimage(hash common.Hash, preimage []byte) {
 	if _, ok := self.preimages[hash]; !ok {
-		self.journal = append(self.journal, addPreimageChange{hash: hash})
+		self.journal.append(addPreimageChange{hash: hash})
 		pi := make([]byte, len(preimage))
 		copy(pi, preimage)
 		self.preimages[hash] = pi
@@ -169,7 +170,7 @@ func (self *StateDB) Preimages() map[common.Hash][]byte {
 }
 
 func (self *StateDB) AddRefund(gas uint64) {
-	self.journal = append(self.journal, refundChange{prev: self.refund})
+	self.journal.append(refundChange{prev: self.refund})
 	self.refund += gas
 }
 
@@ -235,10 +236,10 @@ func (self *StateDB) GetCodeHash(addr common.Address) common.Hash {
 	return common.BytesToHash(stateObject.CodeHash())
 }
 
-func (self *StateDB) GetState(a common.Address, b common.Hash) common.Hash {
+func (self *StateDB) GetState(addr common.Address, bhash common.Hash) common.Hash {
-	stateObject := self.getStateObject(a)
+	stateObject := self.getStateObject(addr)
 	if stateObject != nil {
-		return stateObject.GetState(self.db, b)
+		return stateObject.GetState(self.db, bhash)
 	}
 	return common.Hash{}
 }
@@ -250,12 +251,12 @@ func (self *StateDB) Database() Database {
 
 // StorageTrie returns the storage trie of an account.
 // The return value is a copy and is nil for non-existent accounts.
-func (self *StateDB) StorageTrie(a common.Address) Trie {
+func (self *StateDB) StorageTrie(addr common.Address) Trie {
-	stateObject := self.getStateObject(a)
+	stateObject := self.getStateObject(addr)
 	if stateObject == nil {
 		return nil
 	}
-	cpy := stateObject.deepCopy(self, nil)
+	cpy := stateObject.deepCopy(self)
 	return cpy.updateTrie(self.db)
 }
 
@@ -271,7 +272,7 @@ func (self *StateDB) HasSuicided(addr common.Address) bool {
  * SETTERS
  */
 
-// AddBalance adds amount to the account associated with addr
+// AddBalance adds amount to the account associated with addr.
 func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -279,7 +280,7 @@ func (self *StateDB) AddBalance(addr common.Address, amount *big.Int) {
 	}
 }
 
-// SubBalance subtracts amount from the account associated with addr
+// SubBalance subtracts amount from the account associated with addr.
 func (self *StateDB) SubBalance(addr common.Address, amount *big.Int) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
@@ -308,7 +309,7 @@ func (self *StateDB) SetCode(addr common.Address, code []byte) {
 	}
 }
 
-func (self *StateDB) SetState(addr common.Address, key common.Hash, value common.Hash) {
+func (self *StateDB) SetState(addr common.Address, key, value common.Hash) {
 	stateObject := self.GetOrNewStateObject(addr)
 	if stateObject != nil {
 		stateObject.SetState(self.db, key, value)
@@ -325,7 +326,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 	if stateObject == nil {
 		return false
 	}
-	self.journal = append(self.journal, suicideChange{
+	self.journal.append(suicideChange{
 		account:     &addr,
 		prev:        stateObject.suicided,
 		prevbalance: new(big.Int).Set(stateObject.Balance()),
@@ -337,7 +338,7 @@ func (self *StateDB) Suicide(addr common.Address) bool {
 }
 
 //
-// Setting, updating & deleting state object methods
+// Setting, updating & deleting state object methods.
 //
 
 // updateStateObject writes the given object to the trie.
@@ -379,7 +380,7 @@ func (self *StateDB) getStateObject(addr common.Address) (stateObject *stateObje
 		return nil
 	}
 	// Insert into the live set.
-	obj := newObject(self, addr, data, self.MarkStateObjectDirty)
+	obj := newObject(self, addr, data)
 	self.setStateObject(obj)
 	return obj
 }
@@ -388,7 +389,7 @@ func (self *StateDB) setStateObject(object *stateObject) {
 	self.stateObjects[object.Address()] = object
 }
 
-// Retrieve a state object or create a new state object if nil
+// Retrieve a state object or create a new state object if nil.
 func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	stateObject := self.getStateObject(addr)
 	if stateObject == nil || stateObject.deleted {
@@ -397,22 +398,16 @@ func (self *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 	return stateObject
 }
 
-// MarkStateObjectDirty adds the specified object to the dirty map to avoid costly
-// state object cache iteration to find a handful of modified ones.
-func (self *StateDB) MarkStateObjectDirty(addr common.Address) {
-	self.stateObjectsDirty[addr] = struct{}{}
-}
-
 // createObject creates a new state object. If there is an existing account with
 // the given address, it is overwritten and returned as the second return value.
 func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = self.getStateObject(addr)
-	newobj = newObject(self, addr, Account{}, self.MarkStateObjectDirty)
+	newobj = newObject(self, addr, Account{})
 	newobj.setNonce(0) // sets the object to dirty
 	if prev == nil {
-		self.journal = append(self.journal, createObjectChange{account: &addr})
+		self.journal.append(createObjectChange{account: &addr})
 	} else {
-		self.journal = append(self.journal, resetObjectChange{prev: prev})
+		self.journal.append(resetObjectChange{prev: prev})
 	}
 	self.setStateObject(newobj)
 	return newobj, prev
@@ -466,16 +461,17 @@ func (self *StateDB) Copy() *StateDB {
 	state := &StateDB{
 		db:                self.db,
 		trie:              self.db.CopyTrie(self.trie),
-		stateObjects:      make(map[common.Address]*stateObject, len(self.stateObjectsDirty)),
+		stateObjects:      make(map[common.Address]*stateObject, len(self.journal.dirties)),
-		stateObjectsDirty: make(map[common.Address]struct{}, len(self.stateObjectsDirty)),
+		stateObjectsDirty: make(map[common.Address]struct{}, len(self.journal.dirties)),
 		refund:            self.refund,
 		logs:              make(map[common.Hash][]*types.Log, len(self.logs)),
 		logSize:           self.logSize,
 		preimages:         make(map[common.Hash][]byte),
+		journal:           newJournal(),
 	}
 	// Copy the dirty states, logs, and preimages
-	for addr := range self.stateObjectsDirty {
+	for addr := range self.journal.dirties {
-		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state, state.MarkStateObjectDirty)
+		state.stateObjects[addr] = self.stateObjects[addr].deepCopy(state)
 		state.stateObjectsDirty[addr] = struct{}{}
 	}
 	for hash, logs := range self.logs {
@@ -492,7 +488,7 @@ func (self *StateDB) Copy() *StateDB {
 func (self *StateDB) Snapshot() int {
 	id := self.nextRevisionId
 	self.nextRevisionId++
-	self.validRevisions = append(self.validRevisions, revision{id, len(self.journal)})
+	self.validRevisions = append(self.validRevisions, revision{id, self.journal.length()})
 	return id
 }
 
@@ -507,13 +503,8 @@ func (self *StateDB) RevertToSnapshot(revid int) {
 	}
 	snapshot := self.validRevisions[idx].journalIndex
 
-	// Replay the journal to undo changes.
+	// Replay the journal to undo changes and remove invalidated snapshots
-	for i := len(self.journal) - 1; i >= snapshot; i-- {
+	self.journal.revert(self, snapshot)
-		self.journal[i].undo(self)
-	}
-	self.journal = self.journal[:snapshot]
-
-	// Remove invalidated snapshots from the stack.
 	self.validRevisions = self.validRevisions[:idx]
 }
 
@@ -525,14 +516,25 @@ func (self *StateDB) GetRefund() uint64 {
 // Finalise finalises the state by removing the self destructed objects
 // and clears the journal as well as the refunds.
 func (s *StateDB) Finalise(deleteEmptyObjects bool) {
-	for addr := range s.stateObjectsDirty {
+	for addr := range s.journal.dirties {
-		stateObject := s.stateObjects[addr]
+		stateObject, exist := s.stateObjects[addr]
+		if !exist {
+			// ripeMD is 'touched' at block 1714175, in tx 0x1237f737031e40bcde4a8b7e717b2d15e3ecadfe49bb1bbc71ee9deb09c6fcf2
+			// That tx goes out of gas, and although the notion of 'touched' does not exist there, the
+			// touch-event will still be recorded in the journal. Since ripeMD is a special snowflake,
+			// it will persist in the journal even though the journal is reverted. In this special circumstance,
+			// it may exist in `s.journal.dirties` but not in `s.stateObjects`.
+			// Thus, we can safely ignore it here
+			continue
+		}
+
 		if stateObject.suicided || (deleteEmptyObjects && stateObject.empty()) {
 			s.deleteStateObject(stateObject)
 		} else {
 			stateObject.updateRoot(s.db)
 			s.updateStateObject(stateObject)
 		}
+		s.stateObjectsDirty[addr] = struct{}{}
 	}
 	// Invalidate journal because reverting across transactions is not allowed.
 	s.clearJournalAndRefund()
@@ -576,7 +578,7 @@ func (s *StateDB) DeleteSuicides() {
 }
 
 func (s *StateDB) clearJournalAndRefund() {
-	s.journal = nil
+	s.journal = newJournal()
 	s.validRevisions = s.validRevisions[:0]
 	s.refund = 0
 }
@@ -585,6 +587,9 @@ func (s *StateDB) clearJournalAndRefund() {
 func (s *StateDB) Commit(deleteEmptyObjects bool) (root common.Hash, err error) {
 	defer s.clearJournalAndRefund()
 
+	for addr := range s.journal.dirties {
+		s.stateObjectsDirty[addr] = struct{}{}
+	}
 	// Commit objects to the trie.
 	for addr, stateObject := range s.stateObjects {
 		_, isDirty := s.stateObjectsDirty[addr]

core/state/statedb_test.go

@@ -413,11 +413,12 @@ func (s *StateSuite) TestTouchDelete(c *check.C) {
 
 	snapshot := s.state.Snapshot()
 	s.state.AddBalance(common.Address{}, new(big.Int))
-	if len(s.state.stateObjectsDirty) != 1 {
+
+	if len(s.state.journal.dirties) != 1 {
 		c.Fatal("expected one dirty state object")
 	}
 	s.state.RevertToSnapshot(snapshot)
-	if len(s.state.stateObjectsDirty) != 0 {
+	if len(s.state.journal.dirties) != 0 {
 		c.Fatal("expected no dirty state object")
 	}
 }

core/state_transition.go

@@ -132,28 +132,12 @@ func ApplyMessage(evm *vm.EVM, msg Message, gp *GasPool) ([]byte, uint64, bool,
 	return NewStateTransition(evm, msg, gp).TransitionDb()
 }
 
-func (st *StateTransition) from() vm.AccountRef {
+// to returns the recipient of the message.
-	f := st.msg.From()
+func (st *StateTransition) to() common.Address {
-	if !st.state.Exist(f) {
+	if st.msg == nil || st.msg.To() == nil /* contract creation */ {
-		st.state.CreateAccount(f)
+		return common.Address{}
 	}
-	return vm.AccountRef(f)
+	return *st.msg.To()
-}
-
-func (st *StateTransition) to() vm.AccountRef {
-	if st.msg == nil {
-		return vm.AccountRef{}
-	}
-	to := st.msg.To()
-	if to == nil {
-		return vm.AccountRef{} // contract creation
-	}
-
-	reference := vm.AccountRef(*to)
-	if !st.state.Exist(*to) {
-		st.state.CreateAccount(*to)
-	}
-	return reference
 }
 
 func (st *StateTransition) useGas(amount uint64) error {
@@ -166,12 +150,8 @@ func (st *StateTransition) useGas(amount uint64) error {
 }
 
 func (st *StateTransition) buyGas() error {
-	var (
-		state  = st.state
-		sender = st.from()
-	)
 	mgval := new(big.Int).Mul(new(big.Int).SetUint64(st.msg.Gas()), st.gasPrice)
-	if state.GetBalance(sender.Address()).Cmp(mgval) < 0 {
+	if st.state.GetBalance(st.msg.From()).Cmp(mgval) < 0 {
 		return errInsufficientBalanceForGas
 	}
 	if err := st.gp.SubGas(st.msg.Gas()); err != nil {
@@ -180,20 +160,17 @@ func (st *StateTransition) buyGas() error {
 	st.gas += st.msg.Gas()
 
 	st.initialGas = st.msg.Gas()
-	state.SubBalance(sender.Address(), mgval)
+	st.state.SubBalance(st.msg.From(), mgval)
 	return nil
 }
 
 func (st *StateTransition) preCheck() error {
-	msg := st.msg
+	// Make sure this transaction's nonce is correct.
-	sender := st.from()
+	if st.msg.CheckNonce() {
-
+		nonce := st.state.GetNonce(st.msg.From())
-	// Make sure this transaction's nonce is correct
+		if nonce < st.msg.Nonce() {
-	if msg.CheckNonce() {
-		nonce := st.state.GetNonce(sender.Address())
-		if nonce < msg.Nonce() {
 			return ErrNonceTooHigh
-		} else if nonce > msg.Nonce() {
+		} else if nonce > st.msg.Nonce() {
 			return ErrNonceTooLow
 		}
 	}
@@ -208,8 +185,7 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		return
 	}
 	msg := st.msg
-	sender := st.from() // err checked in preCheck
+	sender := vm.AccountRef(msg.From())
-
 	homestead := st.evm.ChainConfig().IsHomestead(st.evm.BlockNumber)
 	contractCreation := msg.To() == nil
 
@@ -233,8 +209,8 @@ func (st *StateTransition) TransitionDb() (ret []byte, usedGas uint64, failed bo
 		ret, _, st.gas, vmerr = evm.Create(sender, st.data, st.gas, st.value)
 	} else {
 		// Increment the nonce for the next transaction
-		st.state.SetNonce(sender.Address(), st.state.GetNonce(sender.Address())+1)
+		st.state.SetNonce(msg.From(), st.state.GetNonce(sender.Address())+1)
-		ret, st.gas, vmerr = evm.Call(sender, st.to().Address(), st.data, st.gas, st.value)
+		ret, st.gas, vmerr = evm.Call(sender, st.to(), st.data, st.gas, st.value)
 	}
 	if vmerr != nil {
 		log.Debug("VM returned with error", "err", vmerr)
@@ -260,10 +236,8 @@ func (st *StateTransition) refundGas() {
 	st.gas += refund
 
 	// Return ETH for remaining gas, exchanged at the original rate.
-	sender := st.from()
-
 	remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gas), st.gasPrice)
-	st.state.AddBalance(sender.Address(), remaining)
+	st.state.AddBalance(st.msg.From(), remaining)
 
 	// Also return remaining gas to the block gas counter so it is
 	// available for the next transaction.

core/types/block.go

@@ -19,7 +19,6 @@ package types
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 	"math/big"
 	"sort"
@@ -389,40 +388,6 @@ func (b *Block) Hash() common.Hash {
 	return v
 }
 
-func (b *Block) String() string {
-	str := fmt.Sprintf(`Block(#%v): Size: %v {
-MinerHash: %x
-%v
-Transactions:
-%v
-Uncles:
-%v
-}
-`, b.Number(), b.Size(), b.header.HashNoNonce(), b.header, b.transactions, b.uncles)
-	return str
-}
-
-func (h *Header) String() string {
-	return fmt.Sprintf(`Header(%x):
-[
-	ParentHash:	    %x
-	UncleHash:	    %x
-	Coinbase:	    %x
-	Root:		    %x
-	TxSha		    %x
-	ReceiptSha:	    %x
-	Bloom:		    %x
-	Difficulty:	    %v
-	Number:		    %v
-	GasLimit:	    %v
-	GasUsed:	    %v
-	Time:		    %v
-	Extra:		    %s
-	MixDigest:      %x
-	Nonce:		    %x
-]`, h.Hash(), h.ParentHash, h.UncleHash, h.Coinbase, h.Root, h.TxHash, h.ReceiptHash, h.Bloom, h.Difficulty, h.Number, h.GasLimit, h.GasUsed, h.Time, h.Extra, h.MixDigest, h.Nonce)
-}
-
 type Blocks []*Block
 
 type BlockBy func(b1, b2 *Block) bool

core/types/log.go

@@ -17,7 +17,6 @@
 package types
 
 import (
-	"fmt"
 	"io"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -95,10 +94,6 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
 	return err
 }
 
-func (l *Log) String() string {
-	return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
-}
-
 // LogForStorage is a wrapper around a Log that flattens and parses the entire content of
 // a log including non-consensus fields.
 type LogForStorage Log

core/types/receipt.go

@@ -149,14 +149,6 @@ func (r *Receipt) Size() common.StorageSize {
 	return size
 }
 
-// String implements the Stringer interface.
-func (r *Receipt) String() string {
-	if len(r.PostState) == 0 {
-		return fmt.Sprintf("receipt{status=%d cgas=%v bloom=%x logs=%v}", r.Status, r.CumulativeGasUsed, r.Bloom, r.Logs)
-	}
-	return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
-}
-
 // ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
 // entire content of a receipt, as opposed to only the consensus fields originally.
 type ReceiptForStorage Receipt

core/types/transaction.go

@@ -19,7 +19,6 @@ package types
 import (
 	"container/heap"
 	"errors"
-	"fmt"
 	"io"
 	"math/big"
 	"sync/atomic"
@@ -262,58 +261,6 @@ func (tx *Transaction) RawSignatureValues() (*big.Int, *big.Int, *big.Int) {
 	return tx.data.V, tx.data.R, tx.data.S
 }
 
-func (tx *Transaction) String() string {
-	var from, to string
-	if tx.data.V != nil {
-		// make a best guess about the signer and use that to derive
-		// the sender.
-		signer := deriveSigner(tx.data.V)
-		if f, err := Sender(signer, tx); err != nil { // derive but don't cache
-			from = "[invalid sender: invalid sig]"
-		} else {
-			from = fmt.Sprintf("%x", f[:])
-		}
-	} else {
-		from = "[invalid sender: nil V field]"
-	}
-
-	if tx.data.Recipient == nil {
-		to = "[contract creation]"
-	} else {
-		to = fmt.Sprintf("%x", tx.data.Recipient[:])
-	}
-	enc, _ := rlp.EncodeToBytes(&tx.data)
-	return fmt.Sprintf(`
-	TX(%x)
-	Contract: %v
-	From:     %s
-	To:       %s
-	Nonce:    %v
-	GasPrice: %#x
-	GasLimit  %#x
-	Value:    %#x
-	Data:     0x%x
-	V:        %#x
-	R:        %#x
-	S:        %#x
-	Hex:      %x
-`,
-		tx.Hash(),
-		tx.data.Recipient == nil,
-		from,
-		to,
-		tx.data.AccountNonce,
-		tx.data.Price,
-		tx.data.GasLimit,
-		tx.data.Amount,
-		tx.data.Payload,
-		tx.data.V,
-		tx.data.R,
-		tx.data.S,
-		enc,
-	)
-}
-
 // Transactions is a Transaction slice type for basic sorting.
 type Transactions []*Transaction
 

core/vm/logger.go

@@ -45,6 +45,7 @@ type LogConfig struct {
 	DisableMemory  bool // disable memory capture
 	DisableStack   bool // disable stack capture
 	DisableStorage bool // disable storage capture
+	Debug          bool // print output during capture end
 	Limit          int  // maximum length of output, but zero means unlimited
 }
 
@@ -184,6 +185,12 @@ func (l *StructLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost ui
 func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) error {
 	l.output = output
 	l.err = err
+	if l.cfg.Debug {
+		fmt.Printf("0x%x\n", output)
+		if err != nil {
+			fmt.Printf(" error: %v\n", err)
+		}
+	}
 	return nil
 }
 

core/vm/runtime/runtime.go

@@ -103,7 +103,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 		cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(db))
 	}
 	var (
-		address = common.StringToAddress("contract")
+		address = common.BytesToAddress([]byte("contract"))
 		vmenv   = NewEnv(cfg)
 		sender  = vm.AccountRef(cfg.Origin)
 	)
@@ -113,7 +113,7 @@ func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) {
 	// Call the code with the given configuration.
 	ret, _, err := vmenv.Call(
 		sender,
-		common.StringToAddress("contract"),
+		common.BytesToAddress([]byte("contract")),
 		input,
 		cfg.GasLimit,
 		cfg.Value,

crypto/secp256k1/curve.go

@@ -290,11 +290,11 @@ func init() {
 	// See SEC 2 section 2.7.1
 	// curve parameters taken from:
 	// http://www.secg.org/collateral/sec2_final.pdf
-	theCurve.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
+	theCurve.P = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F")
-	theCurve.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
+	theCurve.N = math.MustParseBig256("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141")
-	theCurve.B, _ = new(big.Int).SetString("0000000000000000000000000000000000000000000000000000000000000007", 16)
+	theCurve.B = math.MustParseBig256("0x0000000000000000000000000000000000000000000000000000000000000007")
-	theCurve.Gx, _ = new(big.Int).SetString("79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798", 16)
+	theCurve.Gx = math.MustParseBig256("0x79BE667EF9DCBBAC55A06295CE870B07029BFCDB2DCE28D959F2815B16F81798")
-	theCurve.Gy, _ = new(big.Int).SetString("483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8", 16)
+	theCurve.Gy = math.MustParseBig256("0x483ADA7726A3C4655DA4FBFC0E1108A8FD17B448A68554199C47D08FFB10D4B8")
 	theCurve.BitSize = 256
 }
 

eth/backend.go

@@ -63,7 +63,7 @@ type Ethereum struct {
 	chainConfig *params.ChainConfig
 
 	// Channel for shutting down the service
-	shutdownChan  chan bool    // Channel for shutting down the ethereum
+	shutdownChan  chan bool    // Channel for shutting down the Ethereum
 	stopDbUpgrade func() error // stop chain db sequential key upgrade
 
 	// Handlers
@@ -351,7 +351,7 @@ func (s *Ethereum) StartMining(local bool) error {
 	if local {
 		// If local (CPU) mining is started, we can disable the transaction rejection
 		// mechanism introduced to speed sync times. CPU mining on mainnet is ludicrous
-		// so noone will ever hit this path, whereas marking sync done on CPU mining
+		// so none will ever hit this path, whereas marking sync done on CPU mining
 		// will ensure that private networks work in single miner mode too.
 		atomic.StoreUint32(&s.protocolManager.acceptTxs, 1)
 	}

eth/db_upgrade.go

@@ -62,7 +62,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 			failed    error
 		)
 		for failed == nil && it.Next() {
-			// Skip any entries that don't look like old transaction meta entires (<hash>0x01)
+			// Skip any entries that don't look like old transaction meta entries (<hash>0x01)
 			key := it.Key()
 			if len(key) != common.HashLength+1 || key[common.HashLength] != 0x01 {
 				continue
@@ -86,7 +86,7 @@ func upgradeDeduplicateData(db ethdb.Database) func() error {
 				}
 			}
 			// Convert the old metadata to a new lookup entry, delete duplicate data
-			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new looku entry
+			if failed = db.Put(append([]byte("l"), hash...), it.Value()); failed == nil { // Write the new lookup entry
 				if failed = db.Delete(hash); failed == nil { // Delete the duplicate transaction data
 					if failed = db.Delete(append([]byte("receipts-"), hash...)); failed == nil { // Delete the duplicate receipt data
 						if failed = db.Delete(key); failed != nil { // Delete the old transaction metadata

eth/downloader/downloader.go

@@ -47,7 +47,7 @@ var (
 
 	MaxForkAncestry  = 3 * params.EpochDuration // Maximum chain reorganisation
 	rttMinEstimate   = 2 * time.Second          // Minimum round-trip time to target for download requests
-	rttMaxEstimate   = 20 * time.Second         // Maximum rount-trip time to target for download requests
+	rttMaxEstimate   = 20 * time.Second         // Maximum round-trip time to target for download requests
 	rttMinConfidence = 0.1                      // Worse confidence factor in our estimated RTT value
 	ttlScaling       = 3                        // Constant scaling factor for RTT -> TTL conversion
 	ttlLimit         = time.Minute              // Maximum TTL allowance to prevent reaching crazy timeouts
@@ -884,7 +884,7 @@ func (d *Downloader) fetchHeaders(p *peerConnection, from uint64, pivot uint64)
 // immediately to the header processor to keep the rest of the pipeline full even
 // in the case of header stalls.
 //
-// The method returs the entire filled skeleton and also the number of headers
+// The method returns the entire filled skeleton and also the number of headers
 // already forwarded for processing.
 func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ([]*types.Header, int, error) {
 	log.Debug("Filling up skeleton", "from", from)
@@ -1377,7 +1377,7 @@ func (d *Downloader) processFastSyncContent(latest *types.Header) error {
 		pivot = height - uint64(fsMinFullBlocks)
 	}
 	// To cater for moving pivot points, track the pivot block and subsequently
-	// accumulated download results separatey.
+	// accumulated download results separately.
 	var (
 		oldPivot *fetchResult   // Locked in pivot block, might change eventually
 		oldTail  []*fetchResult // Downloaded content after the pivot
@@ -1615,7 +1615,7 @@ func (d *Downloader) qosReduceConfidence() {
 //
 // Note, the returned RTT is .9 of the actually estimated RTT. The reason is that
 // the downloader tries to adapt queries to the RTT, so multiple RTT values can
-// be adapted to, but smaller ones are preffered (stabler download stream).
+// be adapted to, but smaller ones are preferred (stabler download stream).
 func (d *Downloader) requestRTT() time.Duration {
 	return time.Duration(atomic.LoadUint64(&d.rttEstimate)) * 9 / 10
 }

eth/downloader/downloader_test.go

@@ -159,7 +159,7 @@ func (dl *downloadTester) makeChainFork(n, f int, parent *types.Block, parentRec
 	// Create the common suffix
 	hashes, headers, blocks, receipts := dl.makeChain(n-f, 0, parent, parentReceipts, false)
 
-	// Create the forks, making the second heavyer if non balanced forks were requested
+	// Create the forks, making the second heavier if non balanced forks were requested
 	hashes1, headers1, blocks1, receipts1 := dl.makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]], false)
 	hashes1 = append(hashes1, hashes[1:]...)
 

eth/downloader/fakepeer.go

@@ -27,7 +27,7 @@ import (
 
 // FakePeer is a mock downloader peer that operates on a local database instance
 // instead of being an actual live node. It's useful for testing and to implement
-// sync commands from an xisting local database.
+// sync commands from an existing local database.
 type FakePeer struct {
 	id string
 	db ethdb.Database
@@ -48,7 +48,7 @@ func (p *FakePeer) Head() (common.Hash, *big.Int) {
 }
 
 // RequestHeadersByHash implements downloader.Peer, returning a batch of headers
-// defined by the origin hash and the associaed query parameters.
+// defined by the origin hash and the associated query parameters.
 func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header
@@ -92,7 +92,7 @@ func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int,
 }
 
 // RequestHeadersByNumber implements downloader.Peer, returning a batch of headers
-// defined by the origin number and the associaed query parameters.
+// defined by the origin number and the associated query parameters.
 func (p *FakePeer) RequestHeadersByNumber(number uint64, amount int, skip int, reverse bool) error {
 	var (
 		headers []*types.Header

eth/downloader/peer.go

@@ -551,7 +551,7 @@ func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peerC
 // medianRTT returns the median RTT of the peerset, considering only the tuning
 // peers if there are more peers available.
 func (ps *peerSet) medianRTT() time.Duration {
-	// Gather all the currnetly measured round trip times
+	// Gather all the currently measured round trip times
 	ps.lock.RLock()
 	defer ps.lock.RUnlock()
 

eth/downloader/queue.go

@@ -275,7 +275,7 @@ func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
 	if q.headerResults != nil {
 		panic("skeleton assembly already in progress")
 	}
-	// Shedule all the header retrieval tasks for the skeleton assembly
+	// Schedule all the header retrieval tasks for the skeleton assembly
 	q.headerTaskPool = make(map[uint64]*types.Header)
 	q.headerTaskQueue = prque.New()
 	q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains

eth/downloader/statesync.go

@@ -31,7 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-// stateReq represents a batch of state fetch requests groupped together into
+// stateReq represents a batch of state fetch requests grouped together into
 // a single data retrieval network packet.
 type stateReq struct {
 	items    []common.Hash              // Hashes of the state items to download
@@ -139,7 +139,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 
 		// Handle incoming state packs:
 		case pack := <-d.stateCh:
-			// Discard any data not requested (or previsouly timed out)
+			// Discard any data not requested (or previously timed out)
 			req := active[pack.PeerId()]
 			if req == nil {
 				log.Debug("Unrequested node data", "peer", pack.PeerId(), "len", pack.Items())
@@ -182,7 +182,7 @@ func (d *Downloader) runStateSync(s *stateSync) *stateSync {
 		case req := <-d.trackStateReq:
 			// If an active request already exists for this peer, we have a problem. In
 			// theory the trie node schedule must never assign two requests to the same
-			// peer. In practive however, a peer might receive a request, disconnect and
+			// peer. In practice however, a peer might receive a request, disconnect and
 			// immediately reconnect before the previous times out. In this case the first
 			// request is never honored, alas we must not silently overwrite it, as that
 			// causes valid requests to go missing and sync to get stuck.
@@ -228,7 +228,7 @@ type stateSync struct {
 	err        error          // Any error hit during sync (set before completion)
 }
 
-// stateTask represents a single trie node download taks, containing a set of
+// stateTask represents a single trie node download task, containing a set of
 // peers already attempted retrieval from to detect stalled syncs and abort.
 type stateTask struct {
 	attempts map[string]struct{}
@@ -274,15 +274,21 @@ func (s *stateSync) Cancel() error {
 // receive data from peers, rather those are buffered up in the downloader and
 // pushed here async. The reason is to decouple processing from data receipt
 // and timeouts.
-func (s *stateSync) loop() error {
+func (s *stateSync) loop() (err error) {
 	// Listen for new peer events to assign tasks to them
 	newPeer := make(chan *peerConnection, 1024)
 	peerSub := s.d.peers.SubscribeNewPeers(newPeer)
 	defer peerSub.Unsubscribe()
+	defer func() {
+		cerr := s.commit(true)
+		if err == nil {
+			err = cerr
+		}
+	}()
 
 	// Keep assigning new tasks until the sync completes or aborts
 	for s.sched.Pending() > 0 {
-		if err := s.commit(false); err != nil {
+		if err = s.commit(false); err != nil {
 			return err
 		}
 		s.assignTasks()
@@ -307,14 +313,14 @@ func (s *stateSync) loop() error {
 				s.d.dropPeer(req.peer.id)
 			}
 			// Process all the received blobs and check for stale delivery
-			if err := s.process(req); err != nil {
+			if err = s.process(req); err != nil {
 				log.Warn("Node data write error", "err", err)
 				return err
 			}
 			req.peer.SetNodeDataIdle(len(req.response))
 		}
 	}
-	return s.commit(true)
+	return nil
 }
 
 func (s *stateSync) commit(force bool) error {
@@ -323,7 +329,9 @@ func (s *stateSync) commit(force bool) error {
 	}
 	start := time.Now()
 	b := s.d.stateDB.NewBatch()
-	s.sched.Commit(b)
+	if written, err := s.sched.Commit(b); written == 0 || err != nil {
+		return err
+	}
 	if err := b.Write(); err != nil {
 		return fmt.Errorf("DB write error: %v", err)
 	}
@@ -333,7 +341,7 @@ func (s *stateSync) commit(force bool) error {
 	return nil
 }
 
-// assignTasks attempts to assing new tasks to all idle peers, either from the
+// assignTasks attempts to assign new tasks to all idle peers, either from the
 // batch currently being retried, or fetching new data from the trie sync itself.
 func (s *stateSync) assignTasks() {
 	// Iterate over all idle peers and try to assign them state fetches

eth/fetcher/fetcher.go

@@ -127,7 +127,7 @@ type Fetcher struct {
 	// Block cache
 	queue  *prque.Prque            // Queue containing the import operations (block number sorted)
 	queues map[string]int          // Per peer block counts to prevent memory exhaustion
-	queued map[common.Hash]*inject // Set of already queued blocks (to dedup imports)
+	queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports)
 
 	// Callbacks
 	getBlock       blockRetrievalFn   // Retrieves a block from the local chain

eth/filters/api.go

@@ -98,7 +98,7 @@ func (api *PublicFilterAPI) timeoutLoop() {
 // NewPendingTransactionFilter creates a filter that fetches pending transaction hashes
 // as transactions enter the pending state.
 //
-// It is part of the filter package because this filter can be used throug the
+// It is part of the filter package because this filter can be used through the
 // `eth_getFilterChanges` polling method that is also used for log filters.
 //
 // https://github.com/ethereum/wiki/wiki/JSON-RPC#eth_newpendingtransactionfilter

eth/filters/api_test.go

@@ -29,8 +29,8 @@ func TestUnmarshalJSONNewFilterArgs(t *testing.T) {
 	var (
 		fromBlock rpc.BlockNumber = 0x123435
 		toBlock   rpc.BlockNumber = 0xabcdef
-		address0                  = common.StringToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
+		address0                  = common.HexToAddress("70c87d191324e6712a591f304b4eedef6ad9bb9d")
-		address1                  = common.StringToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
+		address1                  = common.HexToAddress("9b2055d370f73ec7d8a03e965129118dc8f5bf83")
 		topic0                    = common.HexToHash("3ac225168df54212a25c1c01fd35bebfea408fdac2e31ddd6f80a4bbf9a5f1ca")
 		topic1                    = common.HexToHash("9084a792d2f8b16a62b882fd56f7860c07bf5fa91dd8a2ae7e809e5180fef0b3")
 		topic2                    = common.HexToHash("6ccae1c4af4152f460ff510e573399795dfab5dcf1fa60d1f33ac8fdc1e480ce")

eth/handler.go

@@ -96,8 +96,8 @@ type ProtocolManager struct {
 	wg sync.WaitGroup
 }
 
-// NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
+// NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
-// with the ethereum network.
+// with the Ethereum network.
 func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkId uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) {
 	// Create the protocol manager with the base fields
 	manager := &ProtocolManager{
@@ -498,20 +498,20 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
 			return errResp(ErrDecode, "msg %v: %v", msg, err)
 		}
 		// Deliver them all to the downloader for queuing
-		trasactions := make([][]*types.Transaction, len(request))
+		transactions := make([][]*types.Transaction, len(request))
 		uncles := make([][]*types.Header, len(request))
 
 		for i, body := range request {
-			trasactions[i] = body.Transactions
+			transactions[i] = body.Transactions
 			uncles[i] = body.Uncles
 		}
 		// Filter out any explicitly requested bodies, deliver the rest to the downloader
-		filter := len(trasactions) > 0 || len(uncles) > 0
+		filter := len(transactions) > 0 || len(uncles) > 0
 		if filter {
-			trasactions, uncles = pm.fetcher.FilterBodies(p.id, trasactions, uncles, time.Now())
+			transactions, uncles = pm.fetcher.FilterBodies(p.id, transactions, uncles, time.Now())
 		}
-		if len(trasactions) > 0 || len(uncles) > 0 || !filter {
+		if len(transactions) > 0 || len(uncles) > 0 || !filter {
-			err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
+			err := pm.downloader.DeliverBodies(p.id, transactions, uncles)
 			if err != nil {
 				log.Debug("Failed to deliver bodies", "err", err)
 			}

ethclient/ethclient.go

@@ -296,7 +296,7 @@ func (ec *Client) SyncProgress(ctx context.Context) (*ethereum.SyncProgress, err
 // SubscribeNewHead subscribes to notifications about the current blockchain head
 // on the given channel.
 func (ec *Client) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) {
-	return ec.c.EthSubscribe(ctx, ch, "newHeads", map[string]struct{}{})
+	return ec.c.EthSubscribe(ctx, ch, "newHeads")
 }
 
 // State Access

ethdb/database.go

@@ -91,9 +91,6 @@ func (db *LDBDatabase) Path() string {
 
 // Put puts the given key / value to the queue
 func (db *LDBDatabase) Put(key []byte, value []byte) error {
-	// Generate the data to write to disk, update the meter and write
-	//value = rle.Compress(value)
-
 	return db.db.Put(key, value, nil)
 }
 
@@ -103,18 +100,15 @@ func (db *LDBDatabase) Has(key []byte) (bool, error) {
 
 // Get returns the given key if it's present.
 func (db *LDBDatabase) Get(key []byte) ([]byte, error) {
-	// Retrieve the key and increment the miss counter if not found
 	dat, err := db.db.Get(key, nil)
 	if err != nil {
 		return nil, err
 	}
 	return dat, nil
-	//return rle.Decompress(dat)
 }
 
 // Delete deletes the key from the queue and database
 func (db *LDBDatabase) Delete(key []byte) error {
-	// Execute the actual operation
 	return db.db.Delete(key, nil)
 }
 

internal/ethapi/api.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
@@ -1388,7 +1389,7 @@ func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (strin
 	if block == nil {
 		return "", fmt.Errorf("block #%d not found", number)
 	}
-	return block.String(), nil
+	return spew.Sdump(block), nil
 }
 
 // SeedHash retrieves the seed hash of a block.

mobile/types.go

@@ -97,12 +97,6 @@ func (h *Header) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the header for debugging purposes.
-func (h *Header) String() string {
-	return h.header.String()
-}
-
 func (h *Header) GetParentHash() *Hash   { return &Hash{h.header.ParentHash} }
 func (h *Header) GetUncleHash() *Hash    { return &Hash{h.header.UncleHash} }
 func (h *Header) GetCoinbase() *Address  { return &Address{h.header.Coinbase} }
@@ -174,12 +168,6 @@ func (b *Block) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the block for debugging purposes.
-func (b *Block) String() string {
-	return b.block.String()
-}
-
 func (b *Block) GetParentHash() *Hash   { return &Hash{b.block.ParentHash()} }
 func (b *Block) GetUncleHash() *Hash    { return &Hash{b.block.UncleHash()} }
 func (b *Block) GetCoinbase() *Address  { return &Address{b.block.Coinbase()} }
@@ -249,12 +237,6 @@ func (tx *Transaction) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction for debugging purposes.
-func (tx *Transaction) String() string {
-	return tx.tx.String()
-}
-
 func (tx *Transaction) GetData() []byte      { return tx.tx.Data() }
 func (tx *Transaction) GetGas() int64        { return int64(tx.tx.Gas()) }
 func (tx *Transaction) GetGasPrice() *BigInt { return &BigInt{tx.tx.GasPrice()} }
@@ -347,12 +329,6 @@ func (r *Receipt) EncodeJSON() (string, error) {
 	return string(data), err
 }
 
-// String implements the fmt.Stringer interface to print some semi-meaningful
-// data dump of the transaction receipt for debugging purposes.
-func (r *Receipt) String() string {
-	return r.receipt.String()
-}
-
 func (r *Receipt) GetPostState() []byte         { return r.receipt.PostState }
 func (r *Receipt) GetCumulativeGasUsed() int64  { return int64(r.receipt.CumulativeGasUsed) }
 func (r *Receipt) GetBloom() *Bloom             { return &Bloom{r.receipt.Bloom} }

p2p/simulations/adapters/inproc.go

@@ -274,7 +274,7 @@ func (self *SimNode) Start(snapshots map[string][]byte) error {
 		for _, name := range self.config.Services {
 			if err := self.node.Register(newService(name)); err != nil {
 				regErr = err
-				return
+				break
 			}
 		}
 	})

p2p/simulations/mocker.go

@@ -103,8 +103,14 @@ func startStop(net *Network, quit chan struct{}, nodeCount int) {
 func probabilistic(net *Network, quit chan struct{}, nodeCount int) {
 	nodes, err := connectNodesInRing(net, nodeCount)
 	if err != nil {
+		select {
+		case <-quit:
+			//error may be due to abortion of mocking; so the quit channel is closed
+			return
+		default:
 			panic("Could not startup node network for mocker")
 		}
+	}
 	for {
 		select {
 		case <-quit:
@@ -144,7 +150,7 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) {
 			log.Debug(fmt.Sprintf("node %v shutting down", nodes[i]))
 			err := net.Stop(nodes[i])
 			if err != nil {
-				log.Error(fmt.Sprintf("Error stopping node %s", nodes[i]))
+				log.Error("Error stopping node", "node", nodes[i])
 				wg.Done()
 				continue
 			}
@@ -152,7 +158,7 @@ func probabilistic(net *Network, quit chan struct{}, nodeCount int) {
 				time.Sleep(randWait)
 				err := net.Start(id)
 				if err != nil {
-					log.Error(fmt.Sprintf("Error starting node %s", id))
+					log.Error("Error starting node", "node", id)
 				}
 				wg.Done()
 			}(nodes[i])
@@ -169,7 +175,7 @@ func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error)
 		conf := adapters.RandomNodeConfig()
 		node, err := net.NewNodeWithConfig(conf)
 		if err != nil {
-			log.Error("Error creating a node! %s", err)
+			log.Error("Error creating a node!", "err", err)
 			return nil, err
 		}
 		ids[i] = node.ID()
@@ -177,7 +183,7 @@ func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error)
 
 	for _, id := range ids {
 		if err := net.Start(id); err != nil {
-			log.Error("Error starting a node! %s", err)
+			log.Error("Error starting a node!", "err", err)
 			return nil, err
 		}
 		log.Debug(fmt.Sprintf("node %v starting up", id))
@@ -185,7 +191,7 @@ func connectNodesInRing(net *Network, nodeCount int) ([]discover.NodeID, error)
 	for i, id := range ids {
 		peerID := ids[(i+1)%len(ids)]
 		if err := net.Connect(id, peerID); err != nil {
-			log.Error("Error connecting a node to a peer! %s", err)
+			log.Error("Error connecting a node to a peer!", "err", err)
 			return nil, err
 		}
 	}

p2p/simulations/network.go

@@ -87,7 +87,10 @@ func (self *Network) NewNodeWithConfig(conf *adapters.NodeConfig) (*Node, error)
 	if conf.Reachable == nil {
 		conf.Reachable = func(otherID discover.NodeID) bool {
 			_, err := self.InitConn(conf.ID, otherID)
-			return err == nil
+			if err != nil && bytes.Compare(conf.ID.Bytes(), otherID.Bytes()) < 0 {
+				return false
+			}
+			return true
 		}
 	}
 
@@ -448,9 +451,11 @@ func (self *Network) getConn(oneID, otherID discover.NodeID) *Conn {
 // this is cheating as the simulation is used as an oracle and know about
 // remote peers attempt to connect to a node which will then not initiate the connection
 func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) {
+	log.Debug(fmt.Sprintf("InitConn(oneID: %v, otherID: %v)", oneID, otherID))
 	self.lock.Lock()
 	defer self.lock.Unlock()
 	if oneID == otherID {
+		log.Trace(fmt.Sprintf("refusing to connect to self %v", oneID))
 		return nil, fmt.Errorf("refusing to connect to self %v", oneID)
 	}
 	conn, err := self.getOrCreateConn(oneID, otherID)
@@ -458,15 +463,19 @@ func (self *Network) InitConn(oneID, otherID discover.NodeID) (*Conn, error) {
 		return nil, err
 	}
 	if time.Since(conn.initiated) < dialBanTimeout {
+		log.Trace(fmt.Sprintf("connection between %v and %v recently attempted", oneID, otherID))
 		return nil, fmt.Errorf("connection between %v and %v recently attempted", oneID, otherID)
 	}
 	if conn.Up {
+		log.Trace(fmt.Sprintf("%v and %v already connected", oneID, otherID))
 		return nil, fmt.Errorf("%v and %v already connected", oneID, otherID)
 	}
 	err = conn.nodesUp()
 	if err != nil {
+		log.Trace(fmt.Sprintf("nodes not up: %v", err))
 		return nil, fmt.Errorf("nodes not up: %v", err)
 	}
+	log.Debug("InitConn - connection initiated")
 	conn.initiated = time.Now()
 	return conn, nil
 }

swarm/api/api.go

@@ -39,7 +39,8 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}")
+// TODO: this is bad, it should not be hardcoded how long is a hash
+var hashMatcher = regexp.MustCompile("^([0-9A-Fa-f]{64})([0-9A-Fa-f]{64})?")
 
 type ErrResourceReturn struct {
 	key string
@@ -230,20 +231,20 @@ func NewApi(dpa *storage.DPA, dns Resolver, resourceHandler *storage.ResourceHan
 }
 
 // to be used only in TEST
-func (self *Api) Upload(uploadDir, index string) (hash string, err error) {
+func (self *Api) Upload(uploadDir, index string, toEncrypt bool) (hash string, err error) {
 	fs := NewFileSystem(self)
-	hash, err = fs.Upload(uploadDir, index)
+	hash, err = fs.Upload(uploadDir, index, toEncrypt)
 	return hash, err
 }
 
 // DPA reader API
-func (self *Api) Retrieve(key storage.Key) storage.LazySectionReader {
+func (self *Api) Retrieve(key storage.Key) (reader storage.LazySectionReader, isEncrypted bool) {
 	return self.dpa.Retrieve(key)
 }
 
-func (self *Api) Store(data io.Reader, size int64) (key storage.Key, wait func(), err error) {
+func (self *Api) Store(data io.Reader, size int64, toEncrypt bool) (key storage.Key, wait func(), err error) {
 	log.Debug("api.store", "size", size)
-	return self.dpa.Store(data, size, false)
+	return self.dpa.Store(data, size, toEncrypt)
 }
 
 type ErrResolve error
@@ -283,17 +284,17 @@ func (self *Api) Resolve(uri *URI) (storage.Key, error) {
 }
 
 // Put provides singleton manifest creation on top of dpa store
-func (self *Api) Put(content, contentType string) (k storage.Key, wait func(), err error) {
+func (self *Api) Put(content, contentType string, toEncrypt bool) (k storage.Key, wait func(), err error) {
 	apiPutCount.Inc(1)
 	r := strings.NewReader(content)
-	key, waitContent, err := self.dpa.Store(r, int64(len(content)), false)
+	key, waitContent, err := self.dpa.Store(r, int64(len(content)), toEncrypt)
 	if err != nil {
 		apiPutFail.Inc(1)
 		return nil, nil, err
 	}
 	manifest := fmt.Sprintf(`{"entries":[{"hash":"%v","contentType":"%s"}]}`, key, contentType)
 	r = strings.NewReader(manifest)
-	key, waitManifest, err := self.dpa.Store(r, int64(len(manifest)), false)
+	key, waitManifest, err := self.dpa.Store(r, int64(len(manifest)), toEncrypt)
 	if err != nil {
 		apiPutFail.Inc(1)
 		return nil, nil, err
@@ -343,7 +344,7 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe
 		} else {
 			mimeType = entry.ContentType
 			log.Trace("content lookup key", "key", key, "mimetype", mimeType)
-			reader = self.dpa.Retrieve(key)
+			reader, _ = self.dpa.Retrieve(key)
 		}
 	} else {
 		status = http.StatusNotFound
@@ -377,7 +378,7 @@ func (self *Api) Modify(key storage.Key, path, contentHash, contentType string)
 		apiModifyFail.Inc(1)
 		return nil, err
 	}
-	return trie.hash, nil
+	return trie.ref, nil
 }
 
 func (self *Api) AddFile(mhash, path, fname string, content []byte, nameresolver bool) (storage.Key, string, error) {
@@ -481,7 +482,7 @@ func (self *Api) AppendFile(mhash, path, fname string, existingSize int64, conte
 
 	buf := make([]byte, buffSize)
 
-	oldReader := self.Retrieve(oldKey)
+	oldReader, _ := self.Retrieve(oldKey)
 	io.ReadAtLeast(oldReader, buf, int(offset))
 
 	newReader := bytes.NewReader(content)

swarm/api/api_test.go

@@ -32,7 +32,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-func testApi(t *testing.T, f func(*Api)) {
+func testApi(t *testing.T, f func(*Api, bool)) {
 	datadir, err := ioutil.TempDir("", "bzz-test")
 	if err != nil {
 		t.Fatalf("unable to create temp dir: %v", err)
@@ -43,7 +43,8 @@ func testApi(t *testing.T, f func(*Api)) {
 		return
 	}
 	api := NewApi(dpa, nil, nil)
-	f(api)
+	f(api, false)
+	f(api, true)
 }
 
 type testResponse struct {
@@ -106,11 +107,11 @@ func testGet(t *testing.T, api *Api, bzzhash, path string) *testResponse {
 }
 
 func TestApiPut(t *testing.T) {
-	testApi(t, func(api *Api) {
+	testApi(t, func(api *Api, toEncrypt bool) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
 		// exp := expResponse([]byte(content), "text/plain", 0)
-		key, wait, err := api.Put(content, exp.MimeType)
+		key, wait, err := api.Put(content, exp.MimeType, toEncrypt)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}

swarm/api/client/client.go

@@ -52,12 +52,17 @@ type Client struct {
 	Gateway string
 }
 
-// UploadRaw uploads raw data to swarm and returns the resulting hash
+// UploadRaw uploads raw data to swarm and returns the resulting hash. If toEncrypt is true it
-func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
+// uploads encrypted data
+func (c *Client) UploadRaw(r io.Reader, size int64, toEncrypt bool) (string, error) {
 	if size <= 0 {
 		return "", errors.New("data size must be greater than zero")
 	}
-	req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/", r)
+	addr := ""
+	if toEncrypt {
+		addr = "encrypt"
+	}
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz-raw:/"+addr, r)
 	if err != nil {
 		return "", err
 	}
@@ -77,18 +82,20 @@ func (c *Client) UploadRaw(r io.Reader, size int64) (string, error) {
 	return string(data), nil
 }
 
-// DownloadRaw downloads raw data from swarm
+// DownloadRaw downloads raw data from swarm and it returns a ReadCloser and a bool whether the
-func (c *Client) DownloadRaw(hash string) (io.ReadCloser, error) {
+// content was encrypted
+func (c *Client) DownloadRaw(hash string) (io.ReadCloser, bool, error) {
 	uri := c.Gateway + "/bzz-raw:/" + hash
 	res, err := http.DefaultClient.Get(uri)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	if res.StatusCode != http.StatusOK {
 		res.Body.Close()
-		return nil, fmt.Errorf("unexpected HTTP status: %s", res.Status)
+		return nil, false, fmt.Errorf("unexpected HTTP status: %s", res.Status)
 	}
-	return res.Body, nil
+	isEncrypted := (res.Header.Get("X-Decrypted") == "true")
+	return res.Body, isEncrypted, nil
 }
 
 // File represents a file in a swarm manifest and is used for uploading and
@@ -125,11 +132,11 @@ func Open(path string) (*File, error) {
 // (if the manifest argument is non-empty) or creates a new manifest containing
 // the file, returning the resulting manifest hash (the file will then be
 // available at bzz:/<hash>/<path>)
-func (c *Client) Upload(file *File, manifest string) (string, error) {
+func (c *Client) Upload(file *File, manifest string, toEncrypt bool) (string, error) {
 	if file.Size <= 0 {
 		return "", errors.New("file size must be greater than zero")
 	}
-	return c.TarUpload(manifest, &FileUploader{file})
+	return c.TarUpload(manifest, &FileUploader{file}, toEncrypt)
 }
 
 // Download downloads a file with the given path from the swarm manifest with
@@ -159,14 +166,14 @@ func (c *Client) Download(hash, path string) (*File, error) {
 // directory will then be available at bzz:/<hash>/path/to/file), with
 // the file specified in defaultPath being uploaded to the root of the manifest
 // (i.e. bzz:/<hash>/)
-func (c *Client) UploadDirectory(dir, defaultPath, manifest string) (string, error) {
+func (c *Client) UploadDirectory(dir, defaultPath, manifest string, toEncrypt bool) (string, error) {
 	stat, err := os.Stat(dir)
 	if err != nil {
 		return "", err
 	} else if !stat.IsDir() {
 		return "", fmt.Errorf("not a directory: %s", dir)
 	}
-	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath})
+	return c.TarUpload(manifest, &DirectoryUploader{dir, defaultPath}, toEncrypt)
 }
 
 // DownloadDirectory downloads the files contained in a swarm manifest under
@@ -229,26 +236,26 @@ func (c *Client) DownloadDirectory(hash, path, destDir string) error {
 }
 
 // UploadManifest uploads the given manifest to swarm
-func (c *Client) UploadManifest(m *api.Manifest) (string, error) {
+func (c *Client) UploadManifest(m *api.Manifest, toEncrypt bool) (string, error) {
 	data, err := json.Marshal(m)
 	if err != nil {
 		return "", err
 	}
-	return c.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	return c.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
 }
 
 // DownloadManifest downloads a swarm manifest
-func (c *Client) DownloadManifest(hash string) (*api.Manifest, error) {
+func (c *Client) DownloadManifest(hash string) (*api.Manifest, bool, error) {
-	res, err := c.DownloadRaw(hash)
+	res, isEncrypted, err := c.DownloadRaw(hash)
 	if err != nil {
-		return nil, err
+		return nil, isEncrypted, err
 	}
 	defer res.Close()
 	var manifest api.Manifest
 	if err := json.NewDecoder(res).Decode(&manifest); err != nil {
-		return nil, err
+		return nil, isEncrypted, err
 	}
-	return &manifest, nil
+	return &manifest, isEncrypted, nil
 }
 
 // List list files in a swarm manifest which have the given prefix, grouping
@@ -350,10 +357,19 @@ type UploadFn func(file *File) error
 
 // TarUpload uses the given Uploader to upload files to swarm as a tar stream,
 // returning the resulting manifest hash
-func (c *Client) TarUpload(hash string, uploader Uploader) (string, error) {
+func (c *Client) TarUpload(hash string, uploader Uploader, toEncrypt bool) (string, error) {
 	reqR, reqW := io.Pipe()
 	defer reqR.Close()
-	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+hash, reqR)
+	addr := hash
+
+	// If there is a hash already (a manifest), then that manifest will determine if the upload has
+	// to be encrypted or not. If there is no manifest then the toEncrypt parameter decides if
+	// there is encryption or not.
+	if hash == "" && toEncrypt {
+		// This is the built-in address for the encrypted upload endpoint
+		addr = "encrypt"
+	}
+	req, err := http.NewRequest("POST", c.Gateway+"/bzz:/"+addr, reqR)
 	if err != nil {
 		return "", err
 	}

swarm/api/client/client_test.go

@@ -31,6 +31,13 @@ import (
 
 // TestClientUploadDownloadRaw test uploading and downloading raw data to swarm
 func TestClientUploadDownloadRaw(t *testing.T) {
+	testClientUploadDownloadRaw(false, t)
+}
+func TestClientUploadDownloadRawEncrypted(t *testing.T) {
+	testClientUploadDownloadRaw(true, t)
+}
+
+func testClientUploadDownloadRaw(toEncrypt bool, t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
@@ -38,16 +45,19 @@ func TestClientUploadDownloadRaw(t *testing.T) {
 
 	// upload some raw data
 	data := []byte("foo123")
-	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)))
+	hash, err := client.UploadRaw(bytes.NewReader(data), int64(len(data)), toEncrypt)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	// check we can download the same data
-	res, err := client.DownloadRaw(hash)
+	res, isEncrypted, err := client.DownloadRaw(hash)
 	if err != nil {
 		t.Fatal(err)
 	}
+	if isEncrypted != toEncrypt {
+		t.Fatalf("Expected encyption status %v got %v", toEncrypt, isEncrypted)
+	}
 	defer res.Close()
 	gotData, err := ioutil.ReadAll(res)
 	if err != nil {
@@ -61,6 +71,14 @@ func TestClientUploadDownloadRaw(t *testing.T) {
 // TestClientUploadDownloadFiles test uploading and downloading files to swarm
 // manifests
 func TestClientUploadDownloadFiles(t *testing.T) {
+	testClientUploadDownloadFiles(false, t)
+}
+
+func TestClientUploadDownloadFilesEncrypted(t *testing.T) {
+	testClientUploadDownloadFiles(true, t)
+}
+
+func testClientUploadDownloadFiles(toEncrypt bool, t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
@@ -74,7 +92,7 @@ func TestClientUploadDownloadFiles(t *testing.T) {
 				Size:        int64(len(data)),
 			},
 		}
-		hash, err := client.Upload(file, manifest)
+		hash, err := client.Upload(file, manifest, toEncrypt)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -168,7 +186,7 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 	// upload the directory
 	client := NewClient(srv.URL)
 	defaultPath := filepath.Join(dir, testDirFiles[0])
-	hash, err := client.UploadDirectory(dir, defaultPath, "")
+	hash, err := client.UploadDirectory(dir, defaultPath, "", false)
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}
@@ -217,6 +235,14 @@ func TestClientUploadDownloadDirectory(t *testing.T) {
 
 // TestClientFileList tests listing files in a swarm manifest
 func TestClientFileList(t *testing.T) {
+	testClientFileList(false, t)
+}
+
+func TestClientFileListEncrypted(t *testing.T) {
+	testClientFileList(true, t)
+}
+
+func testClientFileList(toEncrypt bool, t *testing.T) {
 	srv := testutil.NewTestSwarmServer(t)
 	defer srv.Close()
 
@@ -224,7 +250,7 @@ func TestClientFileList(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	client := NewClient(srv.URL)
-	hash, err := client.UploadDirectory(dir, "", "")
+	hash, err := client.UploadDirectory(dir, "", "", toEncrypt)
 	if err != nil {
 		t.Fatalf("error uploading directory: %s", err)
 	}

swarm/api/filesystem.go

@@ -47,7 +47,7 @@ func NewFileSystem(api *Api) *FileSystem {
 // TODO: localpath should point to a manifest
 //
 // DEPRECATED: Use the HTTP API instead
-func (self *FileSystem) Upload(lpath, index string) (string, error) {
+func (self *FileSystem) Upload(lpath, index string, toEncrypt bool) (string, error) {
 	var list []*manifestTrieEntry
 	localpath, err := filepath.Abs(filepath.Clean(lpath))
 	if err != nil {
@@ -114,7 +114,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 				stat, _ := f.Stat()
 				var hash storage.Key
 				var wait func()
-				hash, wait, err = self.api.dpa.Store(f, stat.Size(), false)
+				hash, wait, err = self.api.dpa.Store(f, stat.Size(), toEncrypt)
 				if hash != nil {
 					list[i].Hash = hash.Hex()
 				}
@@ -164,7 +164,7 @@ func (self *FileSystem) Upload(lpath, index string) (string, error) {
 	err2 := trie.recalcAndStore()
 	var hs string
 	if err2 == nil {
-		hs = trie.hash.Hex()
+		hs = trie.ref.Hex()
 	}
 	awg.Wait()
 	return hs, err2
@@ -273,7 +273,7 @@ func retrieveToFile(quitC chan bool, dpa *storage.DPA, key storage.Key, path str
 	if err != nil {
 		return err
 	}
-	reader := dpa.Retrieve(key)
+	reader, _ := dpa.Retrieve(key)
 	writer := bufio.NewWriter(f)
 	size, err := reader.Size(quitC)
 	if err != nil {

swarm/api/filesystem_test.go

@@ -29,9 +29,9 @@ import (
 
 var testDownloadDir, _ = ioutil.TempDir(os.TempDir(), "bzz-test")
 
-func testFileSystem(t *testing.T, f func(*FileSystem)) {
+func testFileSystem(t *testing.T, f func(*FileSystem, bool)) {
-	testApi(t, func(api *Api) {
+	testApi(t, func(api *Api, toEncrypt bool) {
-		f(NewFileSystem(api))
+		f(NewFileSystem(api), toEncrypt)
 	})
 }
 
@@ -46,9 +46,9 @@ func readPath(t *testing.T, parts ...string) string {
 }
 
 func TestApiDirUpload0(t *testing.T) {
-	testFileSystem(t, func(fs *FileSystem) {
+	testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
 		api := fs.api
-		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
@@ -74,20 +74,21 @@ func TestApiDirUpload0(t *testing.T) {
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		newbzzhash, err := fs.Upload(downloadDir, "")
+		newbzzhash, err := fs.Upload(downloadDir, "", toEncrypt)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}
-		if bzzhash != newbzzhash {
+		// TODO: currently the hash is not deterministic in the encrypted case
+		if !toEncrypt && bzzhash != newbzzhash {
 			t.Fatalf("download %v reuploaded has incorrect hash, expected %v, got %v", downloadDir, bzzhash, newbzzhash)
 		}
 	})
 }
 
 func TestApiDirUploadModify(t *testing.T) {
-	testFileSystem(t, func(fs *FileSystem) {
+	testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
 		api := fs.api
-		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "")
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "", toEncrypt)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -104,7 +105,7 @@ func TestApiDirUploadModify(t *testing.T) {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)))
+		hash, wait, err := api.Store(bytes.NewReader(index), int64(len(index)), toEncrypt)
 		wait()
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
@@ -144,9 +145,9 @@ func TestApiDirUploadModify(t *testing.T) {
 }
 
 func TestApiDirUploadWithRootFile(t *testing.T) {
-	testFileSystem(t, func(fs *FileSystem) {
+	testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
 		api := fs.api
-		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html")
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0"), "index.html", toEncrypt)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -160,9 +161,9 @@ func TestApiDirUploadWithRootFile(t *testing.T) {
 }
 
 func TestApiFileUpload(t *testing.T) {
-	testFileSystem(t, func(fs *FileSystem) {
+	testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
 		api := fs.api
-		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "")
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "", toEncrypt)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -176,9 +177,9 @@ func TestApiFileUpload(t *testing.T) {
 }
 
 func TestApiFileUploadWithRootFile(t *testing.T) {
-	testFileSystem(t, func(fs *FileSystem) {
+	testFileSystem(t, func(fs *FileSystem, toEncrypt bool) {
 		api := fs.api
-		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html")
+		bzzhash, err := fs.Upload(filepath.Join("testdata", "test0", "index.html"), "index.html", toEncrypt)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return

swarm/api/http/server.go

@@ -124,19 +124,30 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 	log.Debug("handle.post.raw", "ruid", r.ruid)
 
 	postRawCount.Inc(1)
+
+	toEncrypt := false
+	if r.uri.Addr == "encrypt" {
+		toEncrypt = true
+	}
+
 	if r.uri.Path != "" {
 		postRawFail.Inc(1)
 		Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
 		return
 	}
 
+	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
+		postRawFail.Inc(1)
+		Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
+		return
+	}
+
 	if r.Header.Get("Content-Length") == "" {
 		postRawFail.Inc(1)
 		Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
 		return
 	}
-
+	key, _, err := s.api.Store(r.Body, r.ContentLength, toEncrypt)
-	key, _, err := s.api.Store(r.Body, r.ContentLength)
 	if err != nil {
 		postRawFail.Inc(1)
 		Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -166,8 +177,13 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 		return
 	}
 
+	toEncrypt := false
+	if r.uri.Addr == "encrypt" {
+		toEncrypt = true
+	}
+
 	var key storage.Key
-	if r.uri.Addr != "" {
+	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
 		key, err = s.api.Resolve(r.uri)
 		if err != nil {
 			postFilesFail.Inc(1)
@@ -176,7 +192,7 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 		}
 		log.Debug("resolved key", "ruid", r.ruid, "key", key)
 	} else {
-		key, err = s.api.NewManifest()
+		key, err = s.api.NewManifest(toEncrypt)
 		if err != nil {
 			postFilesFail.Inc(1)
 			Respond(w, r, err.Error(), http.StatusInternalServerError)
@@ -540,13 +556,15 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	reader := s.api.Retrieve(key)
+	reader, isEncrypted := s.api.Retrieve(key)
 	if _, err := reader.Size(nil); err != nil {
 		getFail.Inc(1)
 		Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", key, err), http.StatusNotFound)
 		return
 	}
 
+	w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
+
 	switch {
 	case r.uri.Raw() || r.uri.DeprecatedRaw():
 		// allow the request to overwrite the content type using a query
@@ -603,11 +621,12 @@ func (s *Server) HandleGetFiles(w http.ResponseWriter, r *Request) {
 		}
 
 		// retrieve the entry's key and size
-		reader := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
+		reader, isEncrypted := s.api.Retrieve(storage.Key(common.Hex2Bytes(entry.Hash)))
 		size, err := reader.Size(nil)
 		if err != nil {
 			return err
 		}
+		w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
 
 		// write a tar header for the entry
 		hdr := &tar.Header{
@@ -840,14 +859,18 @@ func (s *Server) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
 	req.uri = uri
 
 	log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri", req.uri)
+	log.Debug("parsed request path", "uri.Addr", req.uri.Addr, "uri.path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
 
 	switch r.Method {
 	case "POST":
 		if uri.Raw() || uri.DeprecatedRaw() {
+			log.Debug("handlePostRaw")
 			s.HandlePostRaw(w, req)
 		} else if uri.Resource() {
+			log.Debug("handlePostResource")
 			s.HandlePostResource(w, req)
 		} else {
+			log.Debug("handlePostFiles")
 			s.HandlePostFiles(w, req)
 		}
 

swarm/api/manifest.go

@@ -59,13 +59,13 @@ type ManifestList struct {
 }
 
 // NewManifest creates and stores a new, empty manifest
-func (a *Api) NewManifest() (storage.Key, error) {
+func (a *Api) NewManifest(toEncrypt bool) (storage.Key, error) {
 	var manifest Manifest
 	data, err := json.Marshal(&manifest)
 	if err != nil {
 		return nil, err
 	}
-	key, wait, err := a.Store(bytes.NewReader(data), int64(len(data)))
+	key, wait, err := a.Store(bytes.NewReader(data), int64(len(data)), toEncrypt)
 	wait()
 	return key, err
 }
@@ -83,7 +83,7 @@ func (a *Api) NewResourceManifest(resourceKey string) (storage.Key, error) {
 	if err != nil {
 		return nil, err
 	}
-	key, _, err := a.Store(bytes.NewReader(data), int64(len(data)))
+	key, _, err := a.Store(bytes.NewReader(data), int64(len(data)), false)
 	return key, err
 }
 
@@ -104,7 +104,8 @@ func (a *Api) NewManifestWriter(key storage.Key, quitC chan bool) (*ManifestWrit
 
 // AddEntry stores the given data and adds the resulting key to the manifest
 func (m *ManifestWriter) AddEntry(data io.Reader, e *ManifestEntry) (storage.Key, error) {
-	key, _, err := m.api.Store(data, e.Size)
+
+	key, _, err := m.api.Store(data, e.Size, m.trie.encrypted)
 	if err != nil {
 		return nil, err
 	}
@@ -122,7 +123,7 @@ func (m *ManifestWriter) RemoveEntry(path string) error {
 
 // Store stores the manifest, returning the resulting storage key
 func (m *ManifestWriter) Store() (storage.Key, error) {
-	return m.trie.hash, m.trie.recalcAndStore()
+	return m.trie.ref, m.trie.recalcAndStore()
 }
 
 // ManifestWalker is used to recursively walk the entries in the manifest and
@@ -184,7 +185,8 @@ func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn)
 type manifestTrie struct {
 	dpa       *storage.DPA
 	entries   [257]*manifestTrieEntry // indexed by first character of basePath, entries[256] is the empty basePath entry
-	hash    storage.Key             // if hash != nil, it is stored
+	ref       storage.Key             // if ref != nil, it is stored
+	encrypted bool
 }
 
 func newManifestTrieEntry(entry *ManifestEntry, subtrie *manifestTrie) *manifestTrieEntry {
@@ -203,12 +205,12 @@ type manifestTrieEntry struct {
 func loadManifest(dpa *storage.DPA, hash storage.Key, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 	log.Trace("manifest lookup", "key", hash)
 	// retrieve manifest via DPA
-	manifestReader := dpa.Retrieve(hash)
+	manifestReader, isEncrypted := dpa.Retrieve(hash)
 	log.Trace("reader retrieved", "key", hash)
-	return readManifest(manifestReader, hash, dpa, quitC)
+	return readManifest(manifestReader, hash, dpa, isEncrypted, quitC)
 }
 
-func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
+func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dpa *storage.DPA, isEncrypted bool, quitC chan bool) (trie *manifestTrie, err error) { // non-recursive, subtrees are downloaded on-demand
 
 	// TODO check size for oversized manifests
 	size, err := manifestReader.Size(quitC)
@@ -228,7 +230,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
 		return
 	}
 
-	log.Trace("manifest retrieved", "key", hash)
+	log.Debug("manifest retrieved", "key", hash)
 	var man struct {
 		Entries []*manifestTrieEntry `json:"entries"`
 	}
@@ -243,6 +245,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
 
 	trie = &manifestTrie{
 		dpa:       dpa,
+		encrypted: isEncrypted,
 	}
 	for _, entry := range man.Entries {
 		trie.addEntry(entry, quitC)
@@ -251,7 +254,7 @@ func readManifest(manifestReader storage.LazySectionReader, hash storage.Key, dp
 }
 
 func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
-	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+	self.ref = nil // trie modified, hash needs to be re-calculated on demand
 
 	if len(entry.Path) == 0 {
 		self.entries[256] = entry
@@ -284,6 +287,7 @@ func (self *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 
 	subtrie := &manifestTrie{
 		dpa:       self.dpa,
+		encrypted: self.encrypted,
 	}
 	entry.Path = entry.Path[cpl:]
 	oldentry.Path = oldentry.Path[cpl:]
@@ -307,7 +311,7 @@ func (self *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
 }
 
 func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
-	self.hash = nil // trie modified, hash needs to be re-calculated on demand
+	self.ref = nil // trie modified, hash needs to be re-calculated on demand
 
 	if len(path) == 0 {
 		self.entries[256] = nil
@@ -343,7 +347,7 @@ func (self *manifestTrie) deleteEntry(path string, quitC chan bool) {
 }
 
 func (self *manifestTrie) recalcAndStore() error {
-	if self.hash != nil {
+	if self.ref != nil {
 		return nil
 	}
 
@@ -358,7 +362,7 @@ func (self *manifestTrie) recalcAndStore() error {
 				if err != nil {
 					return err
 				}
-				entry.Hash = entry.subtrie.hash.Hex()
+				entry.Hash = entry.subtrie.ref.Hex()
 			}
 			list.Entries = append(list.Entries, entry.ManifestEntry)
 		}
@@ -371,9 +375,9 @@ func (self *manifestTrie) recalcAndStore() error {
 	}
 
 	sr := bytes.NewReader(manifest)
-	key, wait, err2 := self.dpa.Store(sr, int64(len(manifest)), false)
+	key, wait, err2 := self.dpa.Store(sr, int64(len(manifest)), self.encrypted)
 	wait()
-	self.hash = key
+	self.ref = key
 	return err2
 }
 

swarm/api/manifest_test.go

@@ -42,7 +42,9 @@ func manifest(paths ...string) (manifestReader storage.LazySectionReader) {
 
 func testGetEntry(t *testing.T, path, match string, multiple bool, paths ...string) *manifestTrie {
 	quitC := make(chan bool)
-	trie, err := readManifest(manifest(paths...), nil, nil, quitC)
+	dpa := storage.NewDPA(nil, storage.NewDPAParams())
+	ref := make([]byte, dpa.HashSize())
+	trie, err := readManifest(manifest(paths...), ref, dpa, false, quitC)
 	if err != nil {
 		t.Errorf("unexpected error making manifest: %v", err)
 	}
@@ -97,7 +99,9 @@ func TestGetEntry(t *testing.T) {
 func TestExactMatch(t *testing.T) {
 	quitC := make(chan bool)
 	mf := manifest("shouldBeExactMatch.css", "shouldBeExactMatch.css.map")
-	trie, err := readManifest(mf, nil, nil, quitC)
+	dpa := storage.NewDPA(nil, storage.NewDPAParams())
+	ref := make([]byte, dpa.HashSize())
+	trie, err := readManifest(mf, ref, dpa, false, quitC)
 	if err != nil {
 		t.Errorf("unexpected error making manifest: %v", err)
 	}
@@ -128,7 +132,9 @@ func TestAddFileWithManifestPath(t *testing.T) {
 	reader := &storage.LazyTestSectionReader{
 		SectionReader: io.NewSectionReader(bytes.NewReader(manifest), 0, int64(len(manifest))),
 	}
-	trie, err := readManifest(reader, nil, nil, nil)
+	dpa := storage.NewDPA(nil, storage.NewDPAParams())
+	ref := make([]byte, dpa.HashSize())
+	trie, err := readManifest(reader, ref, dpa, false, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

swarm/api/storage.go

@@ -45,8 +45,8 @@ func NewStorage(api *Api) *Storage {
 // its content type
 //
 // DEPRECATED: Use the HTTP API instead
-func (self *Storage) Put(content, contentType string) (storage.Key, func(), error) {
+func (self *Storage) Put(content, contentType string, toEncrypt bool) (storage.Key, func(), error) {
-	return self.api.Put(content, contentType)
+	return self.api.Put(content, contentType, toEncrypt)
 }
 
 // Get retrieves the content from bzzpath and reads the response in full

swarm/api/storage_test.go

@@ -20,18 +20,18 @@ import (
 	"testing"
 )
 
-func testStorage(t *testing.T, f func(*Storage)) {
+func testStorage(t *testing.T, f func(*Storage, bool)) {
-	testApi(t, func(api *Api) {
+	testApi(t, func(api *Api, toEncrypt bool) {
-		f(NewStorage(api))
+		f(NewStorage(api), toEncrypt)
 	})
 }
 
 func TestStoragePutGet(t *testing.T) {
-	testStorage(t, func(api *Storage) {
+	testStorage(t, func(api *Storage, toEncrypt bool) {
 		content := "hello"
 		exp := expResponse(content, "text/plain", 0)
 		// exp := expResponse([]byte(content), "text/plain", 0)
-		bzzkey, wait, err := api.Put(content, exp.MimeType)
+		bzzkey, wait, err := api.Put(content, exp.MimeType, toEncrypt)
 		if err != nil {
 			t.Fatalf("unexpected error: %v", err)
 		}

swarm/fuse/fuse_file.go

@@ -82,7 +82,7 @@ func (file *SwarmFile) Attr(ctx context.Context, a *fuse.Attr) error {
 	a.Gid = uint32(os.Getegid())
 
 	if file.fileSize == -1 {
-		reader := file.mountInfo.swarmApi.Retrieve(file.key)
+		reader, _ := file.mountInfo.swarmApi.Retrieve(file.key)
 		quitC := make(chan bool)
 		size, err := reader.Size(quitC)
 		if err != nil {
@@ -99,7 +99,7 @@ func (sf *SwarmFile) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse
 	sf.lock.RLock()
 	defer sf.lock.RUnlock()
 	if sf.reader == nil {
-		sf.reader = sf.mountInfo.swarmApi.Retrieve(sf.key)
+		sf.reader, _ = sf.mountInfo.swarmApi.Retrieve(sf.key)
 	}
 	buf := make([]byte, req.Size)
 	n, err := sf.reader.ReadAt(buf, req.Offset)

swarm/fuse/swarmfs_test.go

@@ -38,7 +38,7 @@ type fileInfo struct {
 	contents []byte
 }
 
-func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string) string {
+func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[string]fileInfo, uploadDir string, toEncrypt bool) string {
 	os.RemoveAll(uploadDir)
 
 	for fname, finfo := range files {
@@ -62,9 +62,9 @@ func createTestFilesAndUploadToSwarm(t *testing.T, api *api.Api, files map[strin
 		fd.Close()
 	}
 
-	bzzhash, err := api.Upload(uploadDir, "")
+	bzzhash, err := api.Upload(uploadDir, "", toEncrypt)
 	if err != nil {
-		t.Fatalf("Error uploading directory %v: %v", uploadDir, err)
+		t.Fatalf("Error uploading directory %v: %vm encryption: %v", uploadDir, err, toEncrypt)
 	}
 
 	return bzzhash
@@ -171,7 +171,7 @@ func checkFile(t *testing.T, testMountDir, fname string, contents []byte) {
 	}
 }
 
-func getRandomBtes(size int) []byte {
+func getRandomBytes(size int) []byte {
 	contents := make([]byte, size)
 	rand.Read(contents)
 	return contents
@@ -198,23 +198,25 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T) {
 	testUploadDir, _ := ioutil.TempDir(os.TempDir(), "fuse-source")
 	testMountDir, _ := ioutil.TempDir(os.TempDir(), "fuse-dest")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["2.txt"] = fileInfo{0711, 333, 444, getRandomBtes(10)}
+	files["2.txt"] = fileInfo{0711, 333, 444, getRandomBytes(10)}
-	files["3.txt"] = fileInfo{0622, 333, 444, getRandomBtes(100)}
+	files["3.txt"] = fileInfo{0622, 333, 444, getRandomBytes(100)}
-	files["4.txt"] = fileInfo{0533, 333, 444, getRandomBtes(1024)}
+	files["4.txt"] = fileInfo{0533, 333, 444, getRandomBytes(1024)}
-	files["5.txt"] = fileInfo{0544, 333, 444, getRandomBtes(10)}
+	files["5.txt"] = fileInfo{0544, 333, 444, getRandomBytes(10)}
-	files["6.txt"] = fileInfo{0555, 333, 444, getRandomBtes(10)}
+	files["6.txt"] = fileInfo{0555, 333, 444, getRandomBytes(10)}
-	files["7.txt"] = fileInfo{0666, 333, 444, getRandomBtes(10)}
+	files["7.txt"] = fileInfo{0666, 333, 444, getRandomBytes(10)}
-	files["8.txt"] = fileInfo{0777, 333, 333, getRandomBtes(10)}
+	files["8.txt"] = fileInfo{0777, 333, 333, getRandomBytes(10)}
-	files["11.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
+	files["11.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
-	files["111.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
+	files["111.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
-	files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
+	files["two/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
-	files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
+	files["two/2/2.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
-	files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBtes(10)}
+	files["two/2./2.txt"] = fileInfo{0777, 444, 444, getRandomBytes(10)}
-	files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBtes(200)}
+	files["twice/2.txt"] = fileInfo{0777, 444, 333, getRandomBytes(200)}
-	files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBtes(10240)}
+	files["one/two/three/four/five/six/seven/eight/nine/10.txt"] = fileInfo{0777, 333, 444, getRandomBytes(10240)}
-	files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBtes(10)}
+	files["one/two/three/four/five/six/six"] = fileInfo{0777, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+
+	for _, toEncrypt := range []bool{false, true} {
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs.Stop()
@@ -227,49 +229,55 @@ func (ta *testAPI) mountListAndUnmount(t *testing.T) {
 		if !isDirEmpty(testMountDir) {
 			t.Fatalf("unmount didnt work for %v", testMountDir)
 		}
+	}
 
 }
 
 func (ta *testAPI) maxMounts(t *testing.T) {
+	ta.runMaxMounts(false, t)
+	ta.runMaxMounts(true, t)
+}
+
+func (ta *testAPI) runMaxMounts(toEncrypt bool, t *testing.T) {
 	files := make(map[string]fileInfo)
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir1, _ := ioutil.TempDir(os.TempDir(), "max-upload1")
-	bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1)
+	bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1, toEncrypt)
 	mount1, _ := ioutil.TempDir(os.TempDir(), "max-mount1")
 	swarmfs1 := mountDir(t, ta.api, files, bzzHash1, mount1)
 	defer swarmfs1.Stop()
 
-	files["2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir2, _ := ioutil.TempDir(os.TempDir(), "max-upload2")
-	bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2)
+	bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2, toEncrypt)
 	mount2, _ := ioutil.TempDir(os.TempDir(), "max-mount2")
 	swarmfs2 := mountDir(t, ta.api, files, bzzHash2, mount2)
 	defer swarmfs2.Stop()
 
-	files["3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir3, _ := ioutil.TempDir(os.TempDir(), "max-upload3")
-	bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3)
+	bzzHash3 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir3, toEncrypt)
 	mount3, _ := ioutil.TempDir(os.TempDir(), "max-mount3")
 	swarmfs3 := mountDir(t, ta.api, files, bzzHash3, mount3)
 	defer swarmfs3.Stop()
 
-	files["4.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["4.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir4, _ := ioutil.TempDir(os.TempDir(), "max-upload4")
-	bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4)
+	bzzHash4 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir4, toEncrypt)
 	mount4, _ := ioutil.TempDir(os.TempDir(), "max-mount4")
 	swarmfs4 := mountDir(t, ta.api, files, bzzHash4, mount4)
 	defer swarmfs4.Stop()
 
-	files["5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir5, _ := ioutil.TempDir(os.TempDir(), "max-upload5")
-	bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5)
+	bzzHash5 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir5, toEncrypt)
 	mount5, _ := ioutil.TempDir(os.TempDir(), "max-mount5")
 	swarmfs5 := mountDir(t, ta.api, files, bzzHash5, mount5)
 	defer swarmfs5.Stop()
 
-	files["6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+	files["6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 	uploadDir6, _ := ioutil.TempDir(os.TempDir(), "max-upload6")
-	bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6)
+	bzzHash6 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir6, toEncrypt)
 	mount6, _ := ioutil.TempDir(os.TempDir(), "max-mount6")
 
 	os.RemoveAll(mount6)
@@ -278,20 +286,20 @@ func (ta *testAPI) maxMounts(t *testing.T) {
 	if err == nil {
 		t.Fatalf("Error: Going beyond max mounts  %v", bzzHash6)
 	}
-
 }
 
 func (ta *testAPI) remount(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 		uploadDir1, _ := ioutil.TempDir(os.TempDir(), "re-upload1")
-	bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1)
+		bzzHash1 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir1, toEncrypt)
 		testMountDir1, _ := ioutil.TempDir(os.TempDir(), "re-mount1")
 		swarmfs := mountDir(t, ta.api, files, bzzHash1, testMountDir1)
 		defer swarmfs.Stop()
 
 		uploadDir2, _ := ioutil.TempDir(os.TempDir(), "re-upload2")
-	bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2)
+		bzzHash2 := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir2, toEncrypt)
 		testMountDir2, _ := ioutil.TempDir(os.TempDir(), "re-mount2")
 
 		// try mounting the same hash second time
@@ -314,14 +322,16 @@ func (ta *testAPI) remount(t *testing.T) {
 			t.Fatalf("Error mounting hash  %v", bzzHash2)
 		}
 	}
+}
 
 func (ta *testAPI) unmount(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		uploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, uploadDir, toEncrypt)
 
 		swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs.Stop()
@@ -335,21 +345,23 @@ func (ta *testAPI) unmount(t *testing.T) {
 			}
 		}
 	}
+}
 
 func (ta *testAPI) unmountWhenResourceBusy(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "ex-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "ex-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs.Stop()
 
 		actualPath := filepath.Join(testMountDir, "2.txt")
 		d, err := os.OpenFile(actualPath, os.O_RDWR, os.FileMode(0700))
-	d.Write(getRandomBtes(10))
+		d.Write(getRandomBytes(10))
 
 		_, err = swarmfs.Unmount(testMountDir)
 		if err != nil {
@@ -364,14 +376,16 @@ func (ta *testAPI) unmountWhenResourceBusy(t *testing.T) {
 			}
 		}
 	}
+}
 
 func (ta *testAPI) seekInMultiChunkFile(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "seek-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "seek-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10240)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10240)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs.Stop()
@@ -391,16 +405,18 @@ func (ta *testAPI) seekInMultiChunkFile(t *testing.T) {
 		}
 		d.Close()
 	}
+}
 
 func (ta *testAPI) createNewFile(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "create-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "create-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -428,14 +444,16 @@ func (ta *testAPI) createNewFile(t *testing.T) {
 
 		checkFile(t, testMountDir, "2.txt", contents)
 	}
+}
 
 func (ta *testAPI) createNewFileInsideDirectory(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidedir-mount")
 
-	files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -464,14 +482,16 @@ func (ta *testAPI) createNewFileInsideDirectory(t *testing.T) {
 
 		checkFile(t, testMountDir, "one/2.txt", contents)
 	}
+}
 
 func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "createinsidenewdir-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -501,16 +521,18 @@ func (ta *testAPI) createNewFileInsideNewDirectory(t *testing.T) {
 
 		checkFile(t, testMountDir, "one/2.txt", contents)
 	}
+}
 
 func (ta *testAPI) removeExistingFile(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -529,16 +551,18 @@ func (ta *testAPI) removeExistingFile(t *testing.T) {
 		swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
 		defer swarmfs2.Stop()
 	}
+}
 
 func (ta *testAPI) removeExistingFileInsideDir(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "remove-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "remove-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["one/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["one/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -557,17 +581,18 @@ func (ta *testAPI) removeExistingFileInsideDir(t *testing.T) {
 		swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
 		defer swarmfs2.Stop()
 	}
+}
 
 func (ta *testAPI) removeNewlyAddedFile(t *testing.T) {
-
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "removenew-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "removenew-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -602,16 +627,18 @@ func (ta *testAPI) removeNewlyAddedFile(t *testing.T) {
 			t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest)
 		}
 	}
+}
 
 func (ta *testAPI) addNewFileAndModifyContents(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "modifyfile-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -672,16 +699,18 @@ func (ta *testAPI) addNewFileAndModifyContents(t *testing.T) {
 
 		checkFile(t, testMountDir, "2.txt", line1and2)
 	}
+}
 
 func (ta *testAPI) removeEmptyDir(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount")
 
-	files["1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -696,16 +725,18 @@ func (ta *testAPI) removeEmptyDir(t *testing.T) {
 			t.Fatalf("same contents different hash orig(%v): new(%v)", bzzHash, mi.LatestManifest)
 		}
 	}
+}
 
 func (ta *testAPI) removeDirWhichHasFiles(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmdir-mount")
 
-	files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/five.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/six.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -725,20 +756,22 @@ func (ta *testAPI) removeDirWhichHasFiles(t *testing.T) {
 		swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
 		defer swarmfs2.Stop()
 	}
+}
 
 func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "rmsubdir-mount")
 
-	files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["one/1.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/three/2.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/three/3.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/four/5.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/four/6.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
-	files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBtes(10)}
+		files["two/four/six/7.txt"] = fileInfo{0700, 333, 444, getRandomBytes(10)}
 
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -761,8 +794,10 @@ func (ta *testAPI) removeDirWhichHasSubDirs(t *testing.T) {
 		swarmfs2 := mountDir(t, ta.api, files, mi.LatestManifest, testMountDir)
 		defer swarmfs2.Stop()
 	}
+}
 
 func (ta *testAPI) appendFileContentsToEnd(t *testing.T) {
+	for _, toEncrypt := range []bool{false, true} {
 		files := make(map[string]fileInfo)
 		testUploadDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-upload")
 		testMountDir, _ := ioutil.TempDir(os.TempDir(), "appendlargefile-mount")
@@ -770,7 +805,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T) {
 		line1 := make([]byte, 10)
 		rand.Read(line1)
 		files["1.txt"] = fileInfo{0700, 333, 444, line1}
-	bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir)
+		bzzHash := createTestFilesAndUploadToSwarm(t, ta.api, files, testUploadDir, toEncrypt)
 
 		swarmfs1 := mountDir(t, ta.api, files, bzzHash, testMountDir)
 		defer swarmfs1.Stop()
@@ -800,6 +835,7 @@ func (ta *testAPI) appendFileContentsToEnd(t *testing.T) {
 
 		checkFile(t, testMountDir, "1.txt", line1and2)
 	}
+}
 
 func TestFUSE(t *testing.T) {
 	datadir, err := ioutil.TempDir("", "fuse")

swarm/network/hive.go

@@ -18,7 +18,6 @@ package network
 
 import (
 	"fmt"
-	"math/rand"
 	"sync"
 	"time"
 
@@ -71,7 +70,7 @@ func NewHiveParams() *HiveParams {
 		Discovery:             true,
 		PeersBroadcastSetSize: 3,
 		MaxPeersPerRequest:    5,
-		KeepAliveInterval:     1000 * time.Millisecond,
+		KeepAliveInterval:     500 * time.Millisecond,
 	}
 }
 
@@ -102,10 +101,12 @@ func NewHive(params *HiveParams, overlay Overlay, store state.Store) *Hive {
 // server is used to connect to a peer based on its NodeID or enode URL
 // these are called on the p2p.Server which runs on the node
 func (h *Hive) Start(server *p2p.Server) error {
-	log.Trace(fmt.Sprintf("%08x hive starting", h.BaseAddr()[:4]))
+	log.Info(fmt.Sprintf("%08x hive starting", h.BaseAddr()[:4]))
 	// if state store is specified, load peers to prepopulate the overlay address book
 	if h.Store != nil {
+		log.Info("detected an existing store. trying to load peers")
 		if err := h.loadPeers(); err != nil {
+			log.Error(fmt.Sprintf("%08x hive encoutered an error trying to load peers", h.BaseAddr()[:4]))
 			return err
 		}
 	}
@@ -123,7 +124,12 @@ func (h *Hive) Stop() error {
 	log.Info(fmt.Sprintf("%08x hive stopping, saving peers", h.BaseAddr()[:4]))
 	h.ticker.Stop()
 	if h.Store != nil {
-		return h.savePeers()
+		if err := h.savePeers(); err != nil {
+			return fmt.Errorf("could not save peers to persistence store: %v", err)
+		}
+		if err := h.Store.Close(); err != nil {
+			return fmt.Errorf("could not close file handle to persistence store: %v", err)
+		}
 	}
 	log.Info(fmt.Sprintf("%08x hive stopped, dropping peers", h.BaseAddr()[:4]))
 	h.EachConn(nil, 255, func(p OverlayConn, _ int, _ bool) bool {
@@ -139,8 +145,9 @@ func (h *Hive) Stop() error {
 // at each iteration, ask the overlay driver to suggest the most preferred peer to connect to
 // as well as advertises saturation depth if needed
 func (h *Hive) connect() {
-	time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
 	for range h.ticker.C {
+		log.Trace(fmt.Sprintf("%08x hive connect()", h.BaseAddr()[:4]))
+
 		addr, depth, changed := h.SuggestPeer()
 		if h.Discovery && changed {
 			NotifyDepth(uint8(depth), h)
@@ -203,14 +210,16 @@ func ToAddr(pa OverlayPeer) *BzzAddr {
 // loadPeers, savePeer implement persistence callback/
 func (h *Hive) loadPeers() error {
 	var as []*BzzAddr
-
 	err := h.Store.Get("peers", &as)
 	if err != nil {
 		if err == state.ErrNotFound {
+			log.Info(fmt.Sprintf("hive %08x: no persisted peers found", h.BaseAddr()[:4]))
 			return nil
 		}
 		return err
 	}
+	log.Info(fmt.Sprintf("hive %08x: peers loaded", h.BaseAddr()[:4]))
+
 	return h.Register(toOverlayAddrs(as...))
 }
 

swarm/network/kademlia.go

@@ -24,8 +24,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/p2p/discover"
 	"github.com/ethereum/go-ethereum/pot"
 )
 
@@ -263,11 +263,14 @@ func (k *Kademlia) SuggestPeer() (a OverlayAddr, o int, want bool) {
 		if po >= depth {
 			return false
 		}
-		f(func(val pot.Val, _ int) bool {
+		ok := f(func(val pot.Val, _ int) bool {
 			a = k.callable(val)
 			return a == nil
 		})
+		if !ok {
 			return false
+		}
+		return true
 	})
 	// found a candidate
 	if a != nil {
@@ -616,16 +619,17 @@ type PeerPot struct {
 	EmptyBins []int
 }
 
-// NewPeerPot just creates a new pot record OverlayAddr
+// NewPeerPotMap creates a map of pot record of OverlayAddr with keys
-func NewPeerPot(kadMinProxSize int, ids []discover.NodeID, addrs [][]byte) map[discover.NodeID]*PeerPot {
+// as hexadecimal representations of the address.
+func NewPeerPotMap(kadMinProxSize int, addrs [][]byte) map[string]*PeerPot {
 	// create a table of all nodes for health check
 	np := pot.NewPot(nil, 0)
 	for _, addr := range addrs {
 		np, _, _ = pot.Add(np, addr, pof)
 	}
-	ppmap := make(map[discover.NodeID]*PeerPot)
+	ppmap := make(map[string]*PeerPot)
 
-	for i, id := range ids {
+	for i, a := range addrs {
 		pl := 256
 		prev := 256
 		var emptyBins []int
@@ -654,7 +658,7 @@ func NewPeerPot(kadMinProxSize int, ids []discover.NodeID, addrs [][]byte) map[d
 			emptyBins = append(emptyBins, j)
 		}
 		log.Trace(fmt.Sprintf("%x NNS: %s", addrs[i][:4], logNNS(nns)))
-		ppmap[id] = &PeerPot{nns, emptyBins}
+		ppmap[common.Bytes2Hex(a)] = &PeerPot{nns, emptyBins}
 	}
 	return ppmap
 }
@@ -674,23 +678,38 @@ func (k *Kademlia) saturation(n int) int {
 	return prev
 }
 
+// full returns true if all required bins have connected peers.
+// It is used in Healthy function.
 func (k *Kademlia) full(emptyBins []int) (full bool) {
 	prev := 0
 	e := len(emptyBins)
+	ok := true
+	depth := k.neighbourhoodDepth()
 	k.conns.EachBin(k.base, pof, 0, func(po, _ int, _ func(func(val pot.Val, i int) bool) bool) bool {
-		for i := prev; e > 0 && i < po; i++ {
+		if prev == depth+1 {
+			return true
+		}
+		for i := prev; i < po; i++ {
 			e--
+			if e < 0 {
+				ok = false
+				return false
+			}
 			if emptyBins[e] != i {
 				log.Trace(fmt.Sprintf("%08x po: %d, i: %d, e: %d, emptybins: %v", k.BaseAddr()[:4], po, i, e, logEmptyBins(emptyBins)))
 				if emptyBins[e] < i {
 					panic("incorrect peerpot")
 				}
+				ok = false
 				return false
 			}
 		}
 		prev = po + 1
 		return true
 	})
+	if !ok {
+		return false
+	}
 	return e == 0
 }
 

swarm/network/kademlia_test.go

@@ -17,11 +17,13 @@
 package network
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/pot"
 )
@@ -228,19 +230,18 @@ func TestSuggestPeerFindPeers(t *testing.T) {
 	}
 
 	k.Register("01000001")
-	err = testSuggestPeer(t, k, "<nil>", 0, false)
-	if err != nil {
-		t.Fatal(err.Error())
-	}
-
-	k.On("10000001")
-	log.Trace("Kad:\n%v", k.String())
 	err = testSuggestPeer(t, k, "01000001", 0, false)
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 
 	k.On("10000001")
+	log.Trace(fmt.Sprintf("Kad:\n%v", k.String()))
+	err = testSuggestPeer(t, k, "<nil>", 1, true)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
 	k.On("01000001")
 	err = testSuggestPeer(t, k, "<nil>", 0, false)
 	if err != nil {
@@ -283,7 +284,7 @@ func TestSuggestPeerFindPeers(t *testing.T) {
 func TestSuggestPeerRetries(t *testing.T) {
 	// 2 row gap, unsaturated proxbin, no callables -> want PO 0
 	k := newTestKademlia("00000000")
-	k.RetryInterval = int64(time.Second) // cycle
+	k.RetryInterval = int64(100 * time.Millisecond) // cycle
 	k.MaxRetries = 50
 	k.RetryExponent = 2
 	sleep := func(n int) {
@@ -405,3 +406,266 @@ func TestKademliaHiveString(t *testing.T) {
 		t.Fatalf("incorrect hive output. expected %v, got %v", expH, h)
 	}
 }
+
+// testKademliaCase constructs the kademlia and PeerPot map to validate
+// the SuggestPeer and Healthy methods for provided hex-encoded addresses.
+// Argument pivotAddr is the address of the kademlia.
+func testKademliaCase(t *testing.T, pivotAddr string, addrs ...string) {
+	addr := common.FromHex(pivotAddr)
+	addrs = append(addrs, pivotAddr)
+
+	k := NewKademlia(addr, NewKadParams())
+
+	as := make([][]byte, len(addrs))
+	for i, a := range addrs {
+		as[i] = common.FromHex(a)
+	}
+
+	for _, a := range as {
+		if bytes.Equal(a, addr) {
+			continue
+		}
+		p := &BzzAddr{OAddr: a, UAddr: a}
+		if err := k.Register([]OverlayAddr{p}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	ppmap := NewPeerPotMap(2, as)
+
+	pp := ppmap[pivotAddr]
+
+	for {
+		a, _, _ := k.SuggestPeer()
+		if a == nil {
+			break
+		}
+		k.On(&BzzPeer{BzzAddr: a.(*BzzAddr)})
+	}
+
+	h := k.Healthy(pp)
+	if !(h.GotNN && h.KnowNN && h.Full) {
+		t.Error("not healthy")
+	}
+}
+
+/*
+The regression test for the following invalid kademlia edge case.
+
+Addresses used in this test are discovered as part of the simulation network
+in higher level tests for streaming. They were generated randomly.
+
+=========================================================================
+Mon Apr  9 12:18:24 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 7efef1
+population: 9 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4
+000  2 d7e5 ec56                    | 18 ec56 (0) d7e5 (0) d9e0 (0) c735 (0)
+001  2 18f1 3176                    | 14 18f1 (0) 10bb (0) 10d1 (0) 0421 (0)
+002  2 52aa 47cd                    | 11 52aa (0) 51d9 (0) 5161 (0) 5130 (0)
+003  1 646e                         |  1 646e (0)
+004  0                              |  3 769c (0) 76d1 (0) 7656 (0)
+============ DEPTH: 5 ==========================================
+005  1 7a48                         |  1 7a48 (0)
+006  1 7cbd                         |  1 7cbd (0)
+007  0                              |  0
+008  0                              |  0
+009  0                              |  0
+010  0                              |  0
+011  0                              |  0
+012  0                              |  0
+013  0                              |  0
+014  0                              |  0
+015  0                              |  0
+=========================================================================
+*/
+func TestKademliaCase1(t *testing.T) {
+	testKademliaCase(t,
+		"7efef1c41d77f843ad167be95f6660567eb8a4a59f39240000cce2e0d65baf8e",
+		"ec560e6a4806aa37f147ee83687f3cf044d9953e61eedb8c34b6d50d9e2c5623",
+		"646e9540c84f6a2f9cf6585d45a4c219573b4fd1b64a3c9a1386fc5cf98c0d4d",
+		"18f13c5fba653781019025ab10e8d2fdc916d6448729268afe9e928ffcdbb8e8",
+		"317617acf99b4ffddda8a736f8fc6c6ede0bf690bc23d834123823e6d03e2f69",
+		"d7e52d9647a5d1c27a68c3ee65d543be3947ae4b68537b236d71ef9cb15fb9ab",
+		"7a48f75f8ca60487ae42d6f92b785581b40b91f2da551ae73d5eae46640e02e8",
+		"7cbd42350bde8e18ae5b955b5450f8e2cef3419f92fbf5598160c60fd78619f0",
+		"52aa3ddec61f4d48dd505a2385403c634f6ad06ee1d99c5c90a5ba6006f9af9c",
+		"47cdb6fa93eeb8bc91a417ff4e3b14a9c2ea85137462e2f575fae97f0c4be60d",
+		"5161943eb42e2a03e715fe8afa1009ff5200060c870ead6ab103f63f26cb107f",
+		"a38eaa1255f76bf883ca0830c86e8c4bb7eed259a8348aae9b03f21f90105bee",
+		"b2522bdf1ab26f324e75424fdf6e493b47e8a27687fe76347607b344fc010075",
+		"5bd7213964efb2580b91d02ac31ef126838abeba342f5dbdbe8d4d03562671a2",
+		"0b531adb82744768b694d7f94f73d4f0c9de591266108daeb8c74066bfc9c9ca",
+		"28501f59f70e888d399570145ed884353e017443c675aa12731ada7c87ea14f7",
+		"4a45f1fc63e1a9cb9dfa44c98da2f3d20c2923e5d75ff60b2db9d1bdb0c54d51",
+		"b193431ee35cd32de95805e7c1c749450c47486595aae7195ea6b6019a64fd61",
+		"baebf36a1e35a7ed834e1c72faf44ba16c159fa47d3289ceb3ca35fefa8739b5",
+		"a3659bd32e05fa36c8d20dbaaed8362bf1a8a7bd116aed62d8a43a2efbdf513f",
+		"10d1b50881a4770ebebdd0a75589dabb931e6716747b0f65fd6b080b88c4fdb6",
+		"3c76b8ca5c7ce6a03320646826213f59229626bf5b9d25da0c3ec0662dcb8ff3",
+		"4d72a04ddeb851a68cd197ef9a92a3e2ff01fbbff638e64929dd1a9c2e150112",
+		"c7353d320987956075b5bc1668571c7a36c800d5598fdc4832ec6569561e15d1",
+		"d9e0c7c90878c20ab7639d5954756f54775404b3483407fe1b483635182734f6",
+		"8fca67216b7939c0824fb06c5279901a94da41da9482b000f56df9906736ee75",
+		"460719d7f7aa7d7438f0eaf30333484fa3bd0f233632c10ba89e6e46dd3604be",
+		"0421d92c8a1c79ed5d01305a3d25aaf22a8f5f9e3d4bc80da47ee16ce20465fe",
+		"3441d9d9c0f05820a1bb6459fc7d8ef266a1bd929e7db939a10f544efe8261ea",
+		"ab198a66c293586746758468c610e5d3914d4ce629147eff6dd55a31f863ff8f",
+		"3a1c8c16b0763f3d2c35269f454ff779d1255e954d2deaf6c040fb3f0bcdc945",
+		"5561c0ea3b203e173b11e6aa9d0e621a4e10b1d8b178b8fe375220806557b823",
+		"7656caccdc79cd8d7ce66d415cc96a718e8271c62fb35746bfc2b49faf3eebf3",
+		"5130594fd54c1652cf2debde2c4204573ed76555d1e26757fe345b409af1544a",
+		"76d1e83c71ca246d042e37ff1db181f2776265fbcfdc890ce230bfa617c9c2f0",
+		"89580231962624c53968c1b0095b4a2732b2a2640a19fdd7d21fd064fcc0a5ef",
+		"3d10d001fff44680c7417dd66ecf2e984f0baa20a9bbcea348583ba5ff210c4f",
+		"43754e323f0f3a1155b1852bd6edd55da86b8c4cfe3df8b33733fca50fc202b8",
+		"a9e7b1bb763ae6452ddcacd174993f82977d81a85206bb2ae3c842e2d8e19b4c",
+		"10bb07da7bc7c7757f74149eff167d528a94a253cdc694a863f4d50054c00b6d",
+		"28f0bc1b44658548d6e05dd16d4c2fe77f1da5d48b6774bc4263b045725d0c19",
+		"835fbbf1d16ba7347b6e2fc552d6e982148d29c624ea20383850df3c810fa8fc",
+		"8e236c56a77d7f46e41e80f7092b1a68cd8e92f6156365f41813ad1ca2c6b6f3",
+		"51d9c857e9238c49186e37b4eccf17a82de3d5739f026f6043798ab531456e73",
+		"bbddf7db6a682225301f36a9fd5b0d0121d2951753e1681295f3465352ad511f",
+		"2690a910c33ee37b91eb6c4e0731d1d345e2dc3b46d308503a6e85bbc242c69e",
+		"769ce86aa90b518b7ed382f9fdacfbed93574e18dc98fe6c342e4f9f409c2d5a",
+		"ba3bebec689ce51d3e12776c45f80d25164fdfb694a8122d908081aaa2e7122c",
+		"3a51f4146ea90a815d0d283d1ceb20b928d8b4d45875e892696986a3c0d8fb9b",
+		"81968a2d8fb39114342ee1da85254ec51e0608d7f0f6997c2a8354c260a71009",
+	)
+}
+
+/*
+The regression test for the following invalid kademlia edge case.
+
+Addresses used in this test are discovered as part of the simulation network
+in higher level tests for streaming. They were generated randomly.
+
+=========================================================================
+Mon Apr  9 18:43:48 UTC 2018 KΛÐΞMLIΛ hive: queen's address: bc7f3b
+population: 9 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4
+000  2 0f49 67ff                    | 28 0f49 (0) 0211 (0) 07b2 (0) 0703 (0)
+001  2 e84b f3a4                    | 13 f3a4 (0) e84b (0) e58b (0) e60b (0)
+002  1 8dba                         |  1 8dba (0)
+003  2 a008 ad72                    |  2 ad72 (0) a008 (0)
+004  0                              |  3 b61f (0) b27f (0) b027 (0)
+============ DEPTH: 5 ==========================================
+005  1 ba19                         |  1 ba19 (0)
+006  0                              |  0
+007  1 bdd6                         |  1 bdd6 (0)
+008  0                              |  0
+009  0                              |  0
+010  0                              |  0
+011  0                              |  0
+012  0                              |  0
+013  0                              |  0
+014  0                              |  0
+015  0                              |  0
+=========================================================================
+*/
+func TestKademliaCase2(t *testing.T) {
+	testKademliaCase(t,
+		"bc7f3b6a4a7e3c91b100ca6680b6c06ff407972b88956324ca853295893e0237", "67ffb61d3aa27449d277016188f35f19e2321fbda5008c68cf6303faa080534f", "600cd54c842eadac1729c04abfc369bc244572ca76117105b9dd910283b82730", "d955a05409650de151218557425105a8aa2867bb6a0e0462fa1cf90abcf87ad6", "7a6b726de45abdf7bb3e5fd9fb0dc8932270ca4dedef92238c80c05bcdb570e3", "263e99424ebfdb652adb4e3dcd27d59e11bb7ae1c057b3ef6f390d0228006254", "ba195d1a53aafde68e661c64d39db8c2a73505bf336125c15c3560de3b48b7ed", "3458c762169937115f67cabc35a6c384ed70293a8aec37b077a6c1b8e02d510e", "4ef4dc2e28ac6efdba57e134ac24dd4e0be68b9d54f7006515eb9509105f700c", "2a8782b79b0c24b9714dfd2c8ff1932bebc08aa6520b4eaeaa59ff781238890c", "625d02e960506f4524e9cdeac85b33faf3ea437fceadbd478b62b78720cf24fc", "e051a36a8c8637f520ba259c9ed3fadaf740dadc6a04c3f0e21778ebd4cd6ac4", "e34bc014fa2504f707bb3d904872b56c2fa250bee3cb19a147a0418541f1bd90", "28036dc79add95799916893890add5d8972f3b95325a509d6ded3d448f4dc652", "1b013c407794fa2e4c955d8f51cbc6bd78588a174b6548246b291281304b5409", "34f71b68698e1534095ff23ee9c35bf64c7f12b8463e7c6f6b19c25cf03928b4", "c712c6e9bbb7076832972a95890e340b94ed735935c3c0bb788e61f011b59479", "a008d5becdcda4b9dbfdaafc3cec586cf61dcf2d4b713b6168fff02e3b9f0b08", "29de15555cdbebaab214009e416ee92f947dcec5dab9894129f50f1b17138f34", "5df9449f700bd4b5a23688b68b293f2e92fa6ca524c93bc6bb9936efba9d9ada", "3ab0168a5f87fedc6a39b53c628256ac87a98670d8691bbdaaecec22418d13a2", "1ee299b2d2a74a568494130e6869e66d57982d345c482a0e0eeb285ac219ae3b", "e0e0e3b860cea9b7a74cf1b0675cc632dc64e80a02f20bbc5e96e2e8bb670606", "dc1ba6f169b0fcdcca021dcebaf39fe5d4875e7e69b854fad65687c1d7719ec0", "d321f73e42fcfb1d3a303eddf018ca5dffdcfd5567cd5ec1212f045f6a07e47d", "070320c3da7b542e5ca8aaf6a0a53d2bb5113ed264ab1db2dceee17c729edcb1", "17d314d65fdd136b50d182d2c8f5edf16e7838c2be8cf2c00abe4b406dbcd1d8", "e60b99e0a06f7d2d99d84085f67cdf8cc22a9ae22c339365d80f90289834a2b4", "02115771e18932e1f67a45f11f5bf743c5dae97fbc477d34d35c996012420eac", "3102a40eb2e5060353dd19bf61eeec8782dd1bebfcb57f4c796912252b591827", "8dbaf231062f2dc7ddaba5f9c7761b0c21292be51bf8c2ef503f31d4a2f63f79", "b02787b713c83a9f9183216310f04251994e04c2763a9024731562e8978e7cc4", "b27fe6cd33989e10909ce794c4b0b88feae286b614a59d49a3444c1a7b51ea82", "07b2d2c94fdc6fd148fe23be2ed9eff54f5e12548f29ed8416e6860fc894466f", "e58bf9f451ef62ac44ff0a9bb0610ec0fd14d423235954f0d3695e83017cbfc4", "bdd600b91bb79d1ee0053b854de308cfaa7e2abce575ea6815a0a7b3449609c2", "0f49c93c1edc7999920b21977cedd51a763940dac32e319feb9c1df2da0f3071", "7cbf0297cd41acf655cd6f960d7aaf61479edb4189d5c001cbc730861f0deb41", "79265193778d87ad626a5f59397bc075872d7302a12634ce2451a767d0a82da2", "2fe7d705f7c370b9243dbaafe007d555ff58d218822fca49d347b12a0282457c", "e84bc0c83d05e55a0080eed41dda5a795da4b9313a4da697142e69a65834cbb3", "cc4d278bd9aa0e9fb3cd8d2e0d68fb791aab5de4b120b845c409effbed47a180", "1a2317a8646cd4b6d3c4aa4cc25f676533abb689cf180787db216880a1239ad8", "cbafd6568cf8e99076208e6b6843f5808a7087897c67aad0c54694669398f889", "7b7c8357255fc37b4dae0e1af61589035fd39ff627e0938c6b3da8b4e4ec5d23", "2b8d782c1f5bac46c922cf439f6aa79f91e9ba5ffc0020d58455188a2075b334", "b61f45af2306705740742e76197a119235584ced01ef3f7cf3d4370f6c557cd1", "2775612e7cdae2780bf494c370bdcbe69c55e4a1363b1dc79ea0135e61221cce", "f3a49bb22f40885e961299abfa697a7df690a79f067bf3a4847a3ad48d826c9f", "ad724ac218dc133c0aadf4618eae21fdd0c2f3787af279846b49e2b4f97ff167",
+	)
+}
+
+/*
+The regression test for the following invalid kademlia edge case.
+
+Addresses used in this test are discovered as part of the simulation network
+in higher level tests for streaming. They were generated randomly.
+
+=========================================================================
+Mon Apr  9 19:04:35 UTC 2018 KΛÐΞMLIΛ hive: queen's address: b4822e
+population: 8 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4
+000  2 786c 774b                    | 29 774b (0) 786c (0) 7a79 (0) 7d2f (0)
+001  2 d9de cf19                    | 10 cf19 (0) d9de (0) d2ff (0) d2a2 (0)
+002  2 8ca1 8d74                    |  5 8d74 (0) 8ca1 (0) 9793 (0) 9f51 (0)
+003  0                              |  0
+004  0                              |  3 bfac (0) bcbb (0) bde9 (0)
+005  0                              |  0
+============ DEPTH: 6 ==========================================
+006  1 b660                         |  1 b660 (0)
+007  0                              |  0
+008  1 b450                         |  1 b450 (0)
+009  0                              |  0
+010  0                              |  0
+011  0                              |  0
+012  0                              |  0
+013  0                              |  0
+014  0                              |  0
+015  0                              |  0
+=========================================================================
+*/
+func TestKademliaCase3(t *testing.T) {
+	testKademliaCase(t,
+		"b4822e874a01b94ac3a35c821e6db131e785c2fcbb3556e84b36102caf09b091", "2ecf54ea38d58f9cfc3862e54e5854a7c506fbc640e0b38e46d7d45a19794999", "442374092be50fc7392e8dd3f6fab3158ff7f14f26ff98060aed9b2eecf0b97d", "b450a4a67fcfa3b976cf023d8f1f15052b727f712198ce901630efe2f95db191", "9a7291638eb1c989a6dd6661a42c735b23ac6605b5d3e428aa5ffe650e892c85", "67f62eeab9804cfcac02b25ebeab9113d1b9d03dd5200b1c5a324cc0163e722f", "2e4a0e4b53bca4a9d7e2734150e9f579f29a255ade18a268461b20d026c9ee90", "30dd79c5fcdaa1b106f6960c45c9fde7c046aa3d931088d98c52ab759d0b2ac4", "97936fb5a581e59753c54fa5feec493714f2218245f61f97a62eafd4699433e4", "3a2899b6e129e3e193f6e2aefb82589c948c246d2ec1d4272af32ef3b2660f44", "f0e2a8aa88e67269e9952431ef12e5b29b7f41a1871fbfc38567fad95655d607", "7fa12b3f3c5f8383bfc644b958f72a486969733fa097d8952b3eb4f7b4f73192", "360c167aad5fc992656d6010ec45fdce5bcd492ad9608bc515e2be70d4e430c1", "fe21bc969b3d8e5a64a6484a829c1e04208f26f3cd4de6afcbc172a5bd17f1f1", "b660a1f40141d7ccd282fe5bd9838744119bd1cb3780498b5173578cc5ad308f", "44dcb3370e76680e2fba8cd986ad45ff0b77ca45680ee8d950e47922c4af6226", "8ca126923d17fccb689647307b89f38aa14e2a7b9ebcf3c1e31ccf3d2291a3bc", "f0ae19ae9ce6329327cbf42baf090e084c196b0877d8c7b69997e0123be23ef8", "d2a2a217385158e3e1e348883a14bc423e57daa12077e8c49797d16121ea0810", "f5467ccd85bb4ebe768527db520a210459969a5f1fae6e07b43f519799f0b224", "68be5fd9f9d142a5099e3609011fe3bab7bb992c595999e31e0b3d1668dfb3cf", "4d49a8a476e4934afc6b5c36db9bece3ed1804f20b952da5a21b2b0de766aa73", "ea7155745ef3fb2d099513887a2ba279333ced65c65facbd890ce58bd3fce772", "cf19f51f4e848053d289ac95a9138cdd23fc3077ae913cd58cda8cc7a521b2e1", "590b1cd41c7e6144e76b5cd515a3a4d0a4317624620a3f1685f43ae68bdcd890", "d2ffe0626b5f94a7e00fa0b506e7455a3d9399c15800db108d5e715ef5f6e346", "69630878c50a91f6c2edd23a706bfa0b50bd5661672a37d67bab38e6bca3b698", "445e9067079899bb5faafaca915ae6c0f6b1b730a5a628835dd827636f7feb1e", "6461c77491f1c4825958949f23c153e6e1759a5be53abbcee17c9da3867f3141", "23a235f4083771ccc207771daceda700b525a59ab586788d4f6892e69e34a6e2", "bde99f79ef41a81607ddcf92b9f95dcbc6c3537e91e8bf740e193dc73b19485e", "177957c0e5f0fbd12b88022a91768095d193830986caec8d888097d3ff4310b8", "bcbbdbaa4cdf8352422072f332e05111b732354a35c4d7c617ce1fc3b8b42a5a", "774b6717fdfb0d1629fb9d4c04a9ca40079ae2955d7f82e897477055ed017abb", "16443bf625be6d39ecaa6f114e5d2c1d47a64bfd3c13808d94b55b6b6acef2ee", "8d7495d9008066505ed00ce8198af82bfa5a6b4c08768b4c9fb3aa4eb0b0cca2", "15800849a53349508cb382959527f6c3cf1a46158ff1e6e2316b7dea7967e35f", "7a792f0f4a2b731781d1b244b2a57947f1a2e32900a1c0793449f9f7ae18a7b7", "5e517c2832c9deaa7df77c7bad4d20fd6eda2b7815e155e68bc48238fac1416f", "9f51a14f0019c72bd1d472706d8c80a18c1873c6a0663e754b60eae8094483d7", "7d2fabb565122521d22ba99fed9e5be6a458fbc93156d54db27d97a00b8c3a97", "786c9e412a7db4ec278891fa534caa9a1d1a028c631c6f3aeb9c4d96ad895c36", "3bd6341d40641c2632a5a0cd7a63553a04e251efd7195897a1d27e02a7a8bfde", "31efd1f5fb57b8cff0318d77a1a9e8d67e1d1c8d18ce90f99c3a240dff48cdc8", "d9de3e1156ce1380150948acbcfecd99c96e7f4b0bc97745f4681593d017f74f", "427a2201e09f9583cd990c03b81b58148c297d474a3b50f498d83b1c7a9414cd", "bfaca11596d3dec406a9fcf5d97536516dfe7f0e3b12078428a7e1700e25218a", "351c4770a097248a650008152d0cab5825d048bef770da7f3364f59d1e721bc0", "ee00f205d1486b2be7381d962bd2867263758e880529e4e2bfedfa613bbc0e71", "6aa3b6418d89e3348e4859c823ef4d6d7cd46aa7f7e77aba586c4214d760d8f8",
+	)
+}
+
+/*
+The regression test for the following invalid kademlia edge case.
+
+Addresses used in this test are discovered as part of the simulation network
+in higher level tests for streaming. They were generated randomly.
+
+=========================================================================
+Mon Apr  9 19:16:25 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 9a90fe
+population: 8 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4
+000  2 72ef 4e6c                    | 24 0b1e (0) 0d66 (0) 17f5 (0) 17e8 (0)
+001  2 fc2b fa47                    | 13 fa47 (0) fc2b (0) fffd (0) ecef (0)
+002  2 b847 afa8                    |  6 afa8 (0) ad77 (0) bb7c (0) b847 (0)
+003  0                              |  0
+004  0                              |  4 91fc (0) 957d (0) 9482 (0) 949a (0)
+============ DEPTH: 5 ==========================================
+005  1 9ccf                         |  1 9ccf (0)
+006  0                              |  0
+007  1 9bb2                         |  1 9bb2 (0)
+008  0                              |  0
+009  0                              |  0
+010  0                              |  0
+011  0                              |  0
+012  0                              |  0
+013  0                              |  0
+014  0                              |  0
+015  0                              |  0
+=========================================================================
+*/
+func TestKademliaCase4(t *testing.T) {
+	testKademliaCase(t,
+		"9a90fe3506277244549064b8c3276abb06284a199d9063a97331947f2b7da7f4",
+		"c19359eddef24b7be1a833b4475f212cd944263627a53f9ef4837d106c247730", "fc2b6fef99ef947f7e57c3df376891769e2a2fd83d2b8e634e0fc1e91eaa080c", "ecefc0e1a8ea7bb4b48c469e077401fce175dd75294255b96c4e54f6a2950a55", "bb7ce598efc056bba343cc2614aa3f67a575557561290b44c73a63f8f433f9f7", "55fbee6ca52dfd7f0be0db969ee8e524b654ab4f0cce7c05d83887d7d2a15460", "afa852b6b319998c6a283cc0c82d2f5b8e9410075d7700f3012761f1cfbd0f76", "36c370cfb63f2087971ba6e58d7585b04e16b8f0da335efb91554c2dd8fe191c", "6be41e029985edebc901fb77fc4fb65516b6d85086e2a98bfa3159c99391e585", "dd3cfc72ea553e7d2b28f0037a65646b30955b929d29ba4c40f4a2a811248e77", "da3a8f18e09c7b0ca235c4e33e1441a5188f1df023138bf207753ee63e768f7d", "de9e3ab4dc572d54a2d4b878329fd832bb51a149f4ce167316eeb177b61e7e01", "4e6c1ecde6ed917706257fe020a1d02d2e9d87fca4c85f0f7b132491008c5032", "72ef04b77a070e13463b3529dd312bcacfb7a12d20dc597f5ec3de0501e9b834", "3fef57186675d524ab8bb1f54ba8cb68610babca1247c0c46dbb60aed003c69d", "1d8e6b71f7a052865d6558d4ba44ad5fab7b908cc1badf5766822e1c20d0d823", "6be2f2b4ffa173014d4ec7df157d289744a2bda54bb876b264ccfa898a0da315", "b0ba3fff8643f9985c744327b0c4c869763509fd5da2de9a80a4a0a082021255", "9ccf40b9406ba2e6567101fb9b4e5334a9ec74263eff47267da266ba45e6c158", "d7347f02c180a448e60f73931845062ce00048750b584790278e9c93ef31ad81", "b68c6359a22b3bee6fecb8804311cfd816648ea31d530c9fb48e477e029d707a", "0d668a18ad7c2820214df6df95a6c855ce19fb1cb765f8ca620e45db76686d37", "3fbd2663bff65533246f1fabb9f38086854c6218aeb3dc9ac6ac73d4f0988f91", "949aa5719ca846052bfaa1b38c97b6eca3df3e24c0e0630042c6bccafbb4cdb5", "77b8a2b917bef5d54f3792183b014cca7798f713ff14fe0b2ac79b4c9f6f996d", "17e853cbd8dc00cba3cd9ffeb36f26a9f41a0eb92f80b62c2cda16771c935388", "5f682ed7a8cf2f98387c3def7c97f9f05ae39e39d393eeca3cf621268d6347f8", "ad77487eaf11fd8084ba4517a51766eb0e5b77dd3492dfa79aa3a2802fb29d20", "d247cfcacf9a8200ebaddf639f8c926ab0a001abe682f40df3785e80ed124e91", "195589442e11907eede1ee6524157f1125f68399f3170c835ff81c603b069f6c", "5b5ca0a67f3c54e7d3a6a862ef56168ec9ed1f4945e6c24de6d336b2be2e6f8c", "56430e4caa253015f1f998dce4a48a88af1953f68e94eca14f53074ae9c3e467", "0b1eed6a5bf612d1d8e08f5c546f3d12e838568fd3aa43ed4c537f10c65545d6", "7058db19a56dfff01988ac4a62e1310597f9c8d7ebde6890dadabf047d722d39", "b847380d6888ff7cd11402d086b19eccc40950b52c9d67e73cb4f8462f5df078", "df6c048419a2290ab546d527e9eeba349e7f7e1759bafe4adac507ce60ef9670", "91fc5b4b24fc3fbfea7f9a3d0f0437cb5733c0c2345d8bdffd7048d6e3b8a37b", "957d8ea51b37523952b6f5ae95462fcd4aed1483ef32cc80b69580aaeee03606", "efa82e4e91ad9ab781977400e9ac0bb9de7389aaedebdae979b73d1d3b8d72b0", "7400c9f3f3fc0cc6fe8cc37ab24b9771f44e9f78be913f73cd35fc4be030d6bd", "9bb28f4122d61f7bb56fe27ef706159fb802fef0f5de9dfa32c9c5b3183235f1", "40a8de6e98953498b806614532ea4abf8b99ad7f9719fb68203a6eae2efa5b2a", "412de0b218b8f7dcacc9205cd16ffb4eca5b838f46a2f4f9f534026061a47308", "17f56ecad51075080680ad9faa0fd8946b824d3296ddb20be07f9809fe8d1c5a", "fffd4e7ae885a41948a342b6647955a7ec8a8039039f510cff467ef597675457", "35e78e11b5ac46a29dd04ab0043136c3291f4ca56cb949ace33111ed56395463", "94824fc80230af82077c83bfc01dc9675b1f9d3d538b1e5f41c21ac753598691", "fa470ae314ca3fce493f21b423eef2a49522e09126f6f2326fa3c9cac0b344f7", "7078860b5b621b21ac7b95f9fc4739c8235ce5066a8b9bd7d938146a34fa88ec", "eea53560f0428bfd2eca4f86a5ce9dec5ff1309129a975d73465c1c9e9da71d1",
+	)
+}
+
+/*
+The regression test for the following invalid kademlia edge case.
+
+Addresses used in this test are discovered as part of the simulation network
+in higher level tests for streaming. They were generated randomly.
+
+=========================================================================
+Mon Apr  9 19:25:18 UTC 2018 KΛÐΞMLIΛ hive: queen's address: 5dd5c7
+population: 13 (49), MinProxBinSize: 2, MinBinSize: 2, MaxBinSize: 4
+000  2 e528 fad0                    | 22 fad0 (0) e528 (0) e3bb (0) ed13 (0)
+001  3 3f30 18e0 1dd3               |  7 3f30 (0) 23db (0) 10b6 (0) 18e0 (0)
+002  4 7c54 7804 61e4 60f9          | 10 61e4 (0) 60f9 (0) 636c (0) 7186 (0)
+003  2 40ae 4bae                    |  5 4bae (0) 4d5c (0) 403a (0) 40ae (0)
+004  0                              |  0
+005  0                              |  3 5808 (0) 5a0e (0) 5bdb (0)
+============ DEPTH: 6 ==========================================
+006  2 5f14 5f61                    |  2 5f14 (0) 5f61 (0)
+007  0                              |  0
+008  0                              |  0
+009  0                              |  0
+010  0                              |  0
+011  0                              |  0
+012  0                              |  0
+013  0                              |  0
+014  0                              |  0
+015  0                              |  0
+=========================================================================
+*/
+func TestKademliaCase5(t *testing.T) {
+	testKademliaCase(t,
+		"5dd5c77dd9006a800478fcebb02d48d4036389e7d3c8f6a83b97dbad13f4c0a9",
+		"78fafa0809929a1279ece089a51d12457c2d8416dff859aeb2ccc24bb50df5ec", "1dd39b1257e745f147cbbc3cadd609ccd6207c41056dbc4254bba5d2527d3ee5", "5f61dd66d4d94aec8fcc3ce0e7885c7edf30c43143fa730e2841c5d28e3cd081", "8aa8b0472cb351d967e575ad05c4b9f393e76c4b01ef4b3a54aac5283b78abc9", "4502f385152a915b438a6726ce3ea9342e7a6db91a23c2f6bee83a885ed7eb82", "718677a504249db47525e959ef1784bed167e1c46f1e0275b9c7b588e28a3758", "7c54c6ed1f8376323896ed3a4e048866410de189e9599dd89bf312ca4adb96b5", "18e03bd3378126c09e799a497150da5c24c895aedc84b6f0dbae41fc4bac081a", "23db76ac9e6e58d9f5395ca78252513a7b4118b4155f8462d3d5eec62486cadc", "40ae0e8f065e96c7adb7fa39505136401f01780481e678d718b7f6dbb2c906ec", "c1539998b8bae19d339d6bbb691f4e9daeb0e86847545229e80fe0dffe716e92", "ed139d73a2699e205574c08722ca9f030ad2d866c662f1112a276b91421c3cb9", "5bdb19584b7a36d09ca689422ef7e6bb681b8f2558a6b2177a8f7c812f631022", "636c9de7fe234ffc15d67a504c69702c719f626c17461d3f2918e924cd9d69e2", "de4455413ff9335c440d52458c6544191bd58a16d85f700c1de53b62773064ea", "de1963310849527acabc7885b6e345a56406a8f23e35e436b6d9725e69a79a83", "a80a50a467f561210a114cba6c7fb1489ed43a14d61a9edd70e2eb15c31f074d", "7804f12b8d8e6e4b375b242058242068a3809385e05df0e64973cde805cf729c", "60f9aa320c02c6f2e6370aa740cf7cea38083fa95fca8c99552cda52935c1520", "d8da963602390f6c002c00ce62a84b514edfce9ebde035b277a957264bb54d21", "8463d93256e026fe436abad44697152b9a56ac8e06a0583d318e9571b83d073c", "9a3f78fcefb9a05e40a23de55f6153d7a8b9d973ede43a380bf46bb3b3847de1", "e3bb576f4b3760b9ca6bff59326f4ebfc4a669d263fb7d67ab9797adea54ed13", "4d5cdbd6dcca5bdf819a0fe8d175dc55cc96f088d37462acd5ea14bc6296bdbe", "5a0ed28de7b5258c727cb85447071c74c00a5fbba9e6bc0393bc51944d04ab2a", "61e4ddb479c283c638f4edec24353b6cc7a3a13b930824aad016b0996ca93c47", "7e3610868acf714836cafaaa7b8c009a9ac6e3a6d443e5586cf661530a204ee2", "d74b244d4345d2c86e30a097105e4fb133d53c578320285132a952cdaa64416e", "cfeed57d0f935bfab89e3f630a7c97e0b1605f0724d85a008bbfb92cb47863a8", "580837af95055670e20d494978f60c7f1458dc4b9e389fc7aa4982b2aca3bce3", "df55c0c49e6c8a83d82dfa1c307d3bf6a20e18721c80d8ec4f1f68dc0a137ced", "5f149c51ce581ba32a285439a806c063ced01ccd4211cd024e6a615b8f216f95", "1eb76b00aeb127b10dd1b7cd4c3edeb4d812b5a658f0feb13e85c4d2b7c6fe06", "7a56ba7c3fb7cbfb5561a46a75d95d7722096b45771ec16e6fa7bbfab0b35dfe", "4bae85ad88c28470f0015246d530adc0cd1778bdd5145c3c6b538ee50c4e04bd", "afd1892e2a7145c99ec0ebe9ded0d3fec21089b277a68d47f45961ec5e39e7e0", "953138885d7b36b0ef79e46030f8e61fd7037fbe5ce9e0a94d728e8c8d7eab86", "de761613ef305e4f628cb6bf97d7b7dc69a9d513dc233630792de97bcda777a6", "3f3087280063d09504c084bbf7fdf984347a72b50d097fd5b086ffabb5b3fb4c", "7d18a94bb1ebfdef4d3e454d2db8cb772f30ca57920dd1e402184a9e598581a0", "a7d6fbdc9126d9f10d10617f49fb9f5474ffe1b229f76b7dd27cebba30eccb5d", "fad0246303618353d1387ec10c09ee991eb6180697ed3470ed9a6b377695203d", "1cf66e09ea51ee5c23df26615a9e7420be2ac8063f28f60a3bc86020e94fe6f3", "8269cdaa153da7c358b0b940791af74d7c651cd4d3f5ed13acfe6d0f2c539e7f", "90d52eaaa60e74bf1c79106113f2599471a902d7b1c39ac1f55b20604f453c09", "9788fd0c09190a3f3d0541f68073a2f44c2fcc45bb97558a7c319f36c25a75b3", "10b68fc44157ecfdae238ee6c1ce0333f906ad04d1a4cb1505c8e35c3c87fbb0", "e5284117fdf3757920475c786e0004cb00ba0932163659a89b36651a01e57394", "403ad51d911e113dcd5f9ff58c94f6d278886a2a4da64c3ceca2083282c92de3",
+	)
+}

swarm/network/simulations/discovery/discovery_test.go

@@ -9,10 +9,13 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
+	"path"
+	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p"
@@ -20,6 +23,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/simulations"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/network"
+	"github.com/ethereum/go-ethereum/swarm/state"
 	colorable "github.com/mattn/go-colorable"
 )
 
@@ -27,13 +31,46 @@ import (
 // service to execute
 const serviceName = "discovery"
 const testMinProxBinSize = 2
+const discoveryPersistenceDatadir = "discovery_persistence_test_store"
+
+var discoveryPersistencePath = path.Join(os.TempDir(), discoveryPersistenceDatadir)
+var discoveryEnabled = true
+var persistenceEnabled = false
 
 var services = adapters.Services{
 	serviceName: newService,
 }
 
+func cleanDbStores() error {
+	entries, err := ioutil.ReadDir(os.TempDir())
+	if err != nil {
+		return err
+	}
+
+	for _, f := range entries {
+		if strings.HasPrefix(f.Name(), discoveryPersistenceDatadir) {
+			os.RemoveAll(path.Join(os.TempDir(), f.Name()))
+		}
+	}
+	return nil
+
+}
+
+func getDbStore(nodeID string) (*state.DBStore, error) {
+	if _, err := os.Stat(discoveryPersistencePath + "_" + nodeID); os.IsNotExist(err) {
+		log.Info(fmt.Sprintf("directory for nodeID %s does not exist. creating...", nodeID))
+		ioutil.TempDir("", discoveryPersistencePath+"_"+nodeID)
+	}
+	log.Info(fmt.Sprintf("opening storage directory for nodeID %s", nodeID))
+	store, err := state.NewDBStore(discoveryPersistencePath + "_" + nodeID)
+	if err != nil {
+		return nil, err
+	}
+	return store, nil
+}
+
 var (
-	nodeCount    = flag.Int("nodes", 16, "number of nodes to create (default 10)")
+	nodeCount    = flag.Int("nodes", 10, "number of nodes to create (default 10)")
 	initCount    = flag.Int("conns", 1, "number of originally connected peers	 (default 1)")
 	snapshotFile = flag.String("snapshot", "", "create snapshot")
 	loglevel     = flag.Int("loglevel", 3, "verbosity of logs")
@@ -109,6 +146,14 @@ func TestDiscoverySimulationSimAdapter(t *testing.T) {
 	testDiscoverySimulationSimAdapter(t, *nodeCount, *initCount)
 }
 
+func TestDiscoveryPersistenceSimulationSimAdapter(t *testing.T) {
+	testDiscoveryPersistenceSimulationSimAdapter(t, *nodeCount, *initCount)
+}
+
+func testDiscoveryPersistenceSimulationSimAdapter(t *testing.T, nodes, conns int) {
+	testDiscoveryPersistenceSimulation(t, nodes, conns, adapters.NewSimAdapter(services))
+}
+
 func testDiscoverySimulationSimAdapter(t *testing.T, nodes, conns int) {
 	testDiscoverySimulation(t, nodes, conns, adapters.NewSimAdapter(services))
 }
@@ -144,6 +189,26 @@ func testDiscoverySimulation(t *testing.T, nodes, conns int, adapter adapters.No
 	t.Logf("Setup: %s, shutdown: %s", result.StartedAt.Sub(startedAt), finishedAt.Sub(result.FinishedAt))
 }
 
+func testDiscoveryPersistenceSimulation(t *testing.T, nodes, conns int, adapter adapters.NodeAdapter) map[int][]byte {
+	persistenceEnabled = true
+	discoveryEnabled = true
+
+	result, err := discoveryPersistenceSimulation(nodes, conns, adapter)
+
+	if err != nil {
+		t.Fatalf("Setting up simulation failed: %v", err)
+	}
+	if result.Error != nil {
+		t.Fatalf("Simulation failed: %s", result.Error)
+	}
+	t.Logf("Simulation with %d nodes passed in %s", nodes, result.FinishedAt.Sub(result.StartedAt))
+	// set the discovery and persistence flags again to default so other
+	// tests will not be affected
+	discoveryEnabled = true
+	persistenceEnabled = false
+	return nil
+}
+
 func benchmarkDiscovery(b *testing.B, nodes, conns int) {
 	for i := 0; i < b.N; i++ {
 		result, err := discoverySimulation(nodes, conns, adapters.NewSimAdapter(services))
@@ -207,7 +272,7 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 	wg.Wait()
 	log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
 	// construct the peer pot, so that kademlia health can be checked
-	ppmap := network.NewPeerPot(testMinProxBinSize, ids, addrs)
+	ppmap := network.NewPeerPotMap(testMinProxBinSize, addrs)
 	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
 		select {
 		case <-ctx.Done():
@@ -224,7 +289,8 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 			return false, fmt.Errorf("error getting node client: %s", err)
 		}
 		healthy := &network.Health{}
-		if err := client.Call(&healthy, "hive_healthy", ppmap[id]); err != nil {
+		addr := common.Bytes2Hex(network.ToOverlayAddr(id.Bytes()))
+		if err := client.Call(&healthy, "hive_healthy", ppmap[addr]); err != nil {
 			return false, fmt.Errorf("error getting node health: %s", err)
 		}
 		log.Debug(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v, saturated: %v\n%v", id, healthy.GotNN, healthy.KnowNN, healthy.Full, healthy.Hive))
@@ -266,6 +332,172 @@ func discoverySimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simul
 	return result, nil
 }
 
+func discoveryPersistenceSimulation(nodes, conns int, adapter adapters.NodeAdapter) (*simulations.StepResult, error) {
+	cleanDbStores()
+	defer cleanDbStores()
+
+	// create network
+	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
+		ID:             "0",
+		DefaultService: serviceName,
+	})
+	defer net.Shutdown()
+	trigger := make(chan discover.NodeID)
+	ids := make([]discover.NodeID, nodes)
+	var addrs [][]byte
+
+	for i := 0; i < nodes; i++ {
+		conf := adapters.RandomNodeConfig()
+		node, err := net.NewNodeWithConfig(conf)
+		if err != nil {
+			panic(err)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("error starting node: %s", err)
+		}
+		if err := net.Start(node.ID()); err != nil {
+			return nil, fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err)
+		}
+		if err := triggerChecks(trigger, net, node.ID()); err != nil {
+			return nil, fmt.Errorf("error triggering checks for node %s: %s", node.ID().TerminalString(), err)
+		}
+		ids[i] = node.ID()
+		a := network.ToOverlayAddr(ids[i].Bytes())
+
+		addrs = append(addrs, a)
+	}
+
+	// run a simulation which connects the 10 nodes in a ring and waits
+	// for full peer discovery
+	ppmap := network.NewPeerPotMap(testMinProxBinSize, addrs)
+
+	var restartTime time.Time
+
+	action := func(ctx context.Context) error {
+		ticker := time.NewTicker(500 * time.Millisecond)
+
+		for range ticker.C {
+			isHealthy := true
+			for _, id := range ids {
+				//call Healthy RPC
+				node := net.GetNode(id)
+				if node == nil {
+					return fmt.Errorf("unknown node: %s", id)
+				}
+				client, err := node.Client()
+				if err != nil {
+					return fmt.Errorf("error getting node client: %s", err)
+				}
+				healthy := &network.Health{}
+				addr := common.Bytes2Hex(network.ToOverlayAddr(id.Bytes()))
+				if err := client.Call(&healthy, "hive_healthy", ppmap[addr]); err != nil {
+					return fmt.Errorf("error getting node health: %s", err)
+				}
+
+				log.Info(fmt.Sprintf("NODE: %s, IS HEALTHY: %t", id.String(), healthy.GotNN && healthy.KnowNN && healthy.Full))
+				if !healthy.GotNN || !healthy.Full {
+					isHealthy = false
+					break
+				}
+			}
+			if isHealthy {
+				break
+			}
+		}
+		ticker.Stop()
+
+		log.Info("reached healthy kademlia. starting to shutdown nodes.")
+		shutdownStarted := time.Now()
+		// stop all ids, then start them again
+		for _, id := range ids {
+			node := net.GetNode(id)
+
+			if err := net.Stop(node.ID()); err != nil {
+				return fmt.Errorf("error stopping node %s: %s", node.ID().TerminalString(), err)
+			}
+		}
+		log.Info(fmt.Sprintf("shutting down nodes took: %s", time.Now().Sub(shutdownStarted)))
+		persistenceEnabled = true
+		discoveryEnabled = false
+		restartTime = time.Now()
+		for _, id := range ids {
+			node := net.GetNode(id)
+			if err := net.Start(node.ID()); err != nil {
+				return fmt.Errorf("error starting node %s: %s", node.ID().TerminalString(), err)
+			}
+			if err := triggerChecks(trigger, net, node.ID()); err != nil {
+				return fmt.Errorf("error triggering checks for node %s: %s", node.ID().TerminalString(), err)
+			}
+		}
+
+		log.Info(fmt.Sprintf("restarting nodes took: %s", time.Now().Sub(restartTime)))
+
+		return nil
+	}
+	//connects in a chain
+	wg := sync.WaitGroup{}
+	//connects in a ring
+	for i := range ids {
+		for j := 1; j <= conns; j++ {
+			k := (i + j) % len(ids)
+			if k == i {
+				k = (k + 1) % len(ids)
+			}
+			wg.Add(1)
+			go func(i, k int) {
+				defer wg.Done()
+				net.Connect(ids[i], ids[k])
+			}(i, k)
+		}
+	}
+	wg.Wait()
+	log.Debug(fmt.Sprintf("nodes: %v", len(addrs)))
+	// construct the peer pot, so that kademlia health can be checked
+	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		default:
+		}
+
+		node := net.GetNode(id)
+		if node == nil {
+			return false, fmt.Errorf("unknown node: %s", id)
+		}
+		client, err := node.Client()
+		if err != nil {
+			return false, fmt.Errorf("error getting node client: %s", err)
+		}
+		healthy := &network.Health{}
+		addr := common.Bytes2Hex(network.ToOverlayAddr(id.Bytes()))
+		if err := client.Call(&healthy, "hive_healthy", ppmap[addr]); err != nil {
+			return false, fmt.Errorf("error getting node health: %s", err)
+		}
+		log.Info(fmt.Sprintf("node %4s healthy: got nearest neighbours: %v, know nearest neighbours: %v, saturated: %v", id, healthy.GotNN, healthy.KnowNN, healthy.Full))
+
+		return healthy.KnowNN && healthy.GotNN && healthy.Full, nil
+	}
+
+	// 64 nodes ~ 1min
+	// 128 nodes ~
+	timeout := 300 * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+	result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
+		Action:  action,
+		Trigger: trigger,
+		Expect: &simulations.Expectation{
+			Nodes: ids,
+			Check: check,
+		},
+	})
+	if result.Error != nil {
+		return result, nil
+	}
+
+	return result, nil
+}
+
 // triggerChecks triggers a simulation step check whenever a peer is added or
 // removed from the given node, and also every second to avoid a race between
 // peer events and kademlia becoming healthy
@@ -313,11 +545,6 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) {
 
 	kp := network.NewKadParams()
 	kp.MinProxBinSize = testMinProxBinSize
-	kp.MaxBinSize = 3
-	kp.MinBinSize = 1
-	kp.MaxRetries = 1000
-	kp.RetryExponent = 2
-	kp.RetryInterval = 50000000
 
 	if ctx.Config.Reachable != nil {
 		kp.Reachable = func(o network.OverlayAddr) bool {
@@ -325,9 +552,11 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) {
 		}
 	}
 	kad := network.NewKademlia(addr.Over(), kp)
-
 	hp := network.NewHiveParams()
-	hp.KeepAliveInterval = 200 * time.Millisecond
+	hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
+	hp.Discovery = discoveryEnabled
+
+	log.Info(fmt.Sprintf("discovery for nodeID %s is %t", ctx.Config.ID.String(), hp.Discovery))
 
 	config := &network.BzzConfig{
 		OverlayAddr:  addr.Over(),
@@ -335,5 +564,14 @@ func newService(ctx *adapters.ServiceContext) (node.Service, error) {
 		HiveParams:   hp,
 	}
 
+	if persistenceEnabled {
+		log.Info(fmt.Sprintf("persistence enabled for nodeID %s", ctx.Config.ID.String()))
+		store, err := getDbStore(ctx.Config.ID.String())
+		if err != nil {
+			return nil, err
+		}
+		return network.NewBzz(config, kad, store, nil, nil), nil
+	}
+
 	return network.NewBzz(config, kad, nil, nil, nil), nil
 }

swarm/network/simulations/overlay.go

@@ -7,8 +7,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"math/rand"
 	"net/http"
 	"os"
 	"runtime"
@@ -72,162 +70,23 @@ func (s *Simulation) NewService(ctx *adapters.ServiceContext) (node.Service, err
 	return network.NewBzz(config, kad, store, nil, nil), nil
 }
 
-func createMockers() map[string]*simulations.MockerConfig {
-	configs := make(map[string]*simulations.MockerConfig)
-
-	defaultCfg := simulations.DefaultMockerConfig()
-	defaultCfg.ID = "start-stop"
-	defaultCfg.Description = "Starts and Stops nodes in go routines"
-	defaultCfg.Mocker = startStopMocker
-
-	bootNetworkCfg := simulations.DefaultMockerConfig()
-	bootNetworkCfg.ID = "bootNet"
-	bootNetworkCfg.Description = "Only boots up all nodes in the config"
-	bootNetworkCfg.Mocker = bootMocker
-
-	randomNodesCfg := simulations.DefaultMockerConfig()
-	randomNodesCfg.ID = "randomNodes"
-	randomNodesCfg.Description = "Boots nodes and then starts and stops some picking randomly"
-	randomNodesCfg.Mocker = randomMocker
-
-	configs[defaultCfg.ID] = defaultCfg
-	configs[bootNetworkCfg.ID] = bootNetworkCfg
-	configs[randomNodesCfg.ID] = randomNodesCfg
-
-	return configs
-}
-
-func setupMocker(net *simulations.Network) []discover.NodeID {
-	nodeCount := 30
-	ids := make([]discover.NodeID, nodeCount)
-	for i := 0; i < nodeCount; i++ {
-		node, err := net.NewNode()
-		if err != nil {
-			panic(err.Error())
-		}
-		ids[i] = node.ID()
-	}
-
-	for _, id := range ids {
-		if err := net.Start(id); err != nil {
-			panic(err.Error())
-		}
-	}
-	for i, id := range ids {
-		log.Trace(fmt.Sprintf("setup mocker: register a peer on node %x", id[:4]))
-		var peerID discover.NodeID
-		if i == 0 {
-			peerID = ids[len(ids)-1]
-		} else {
-			peerID = ids[i-1]
-		}
-		ch := make(chan network.OverlayAddr)
-		go func() {
-			defer close(ch)
-			ch <- network.NewAddrFromNodeID(peerID)
-		}()
-		log.Trace(fmt.Sprintf("%x registers peer %x", id[:4], peerID[:4]))
-		if err := net.GetNode(id).Node.(*adapters.SimNode).Services()[0].(*network.Bzz).Hive.Register(ch); err != nil {
-			panic(err.Error())
-		}
-	}
-
-	return ids
-}
-
-func bootMocker(net *simulations.Network) {
-	setupMocker(net)
-}
-
-func randomMocker(net *simulations.Network) {
-	ids := setupMocker(net)
-
-	for {
-		var lowid, highid int
-		var wg sync.WaitGroup
-		randWait := rand.Intn(5000) + 1000
-		rand1 := rand.Intn(9)
-		rand2 := rand.Intn(9)
-		if rand1 < rand2 {
-			lowid = rand1
-			highid = rand2
-		} else if rand1 > rand2 {
-			highid = rand1
-			lowid = rand2
-		} else {
-			if rand1 == 0 {
-				rand2 = 9
-			} else if rand1 == 9 {
-				rand1 = 0
-			}
-			lowid = rand1
-			highid = rand2
-		}
-		var steps = highid - lowid
-		wg.Add(steps)
-		for i := lowid; i < highid; i++ {
-			log.Info(fmt.Sprintf("node %v shutting down", ids[i]))
-			net.Stop(ids[i])
-			go func(id discover.NodeID) {
-				time.Sleep(time.Duration(randWait) * time.Millisecond)
-				net.Start(id)
-				wg.Done()
-			}(ids[i])
-			time.Sleep(time.Duration(randWait) * time.Millisecond)
-		}
-		wg.Wait()
-	}
-}
-
-func startStopMocker(net *simulations.Network) {
-	ids := setupMocker(net)
-
-	for range time.Tick(10 * time.Second) {
-		id := ids[rand.Intn(len(ids))]
-		go func() {
-			log.Error("stopping node", "id", id)
-			if err := net.Stop(id); err != nil {
-				log.Error("error stopping node", "id", id, "err", err)
-				return
-			}
-
-			time.Sleep(3 * time.Second)
-
-			log.Error("starting node", "id", id)
-			if err := net.Start(id); err != nil {
-				log.Error("error starting node", "id", id, "err", err)
-				return
-			}
-		}()
-	}
-}
-
 // var server
 func main() {
 	flag.Parse()
 
 	runtime.GOMAXPROCS(runtime.NumCPU())
 
-	log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(false))))
 
 	s := NewSimulation()
 	services := adapters.Services{
 		"overlay": s.NewService,
 	}
 	adapter := adapters.NewSimAdapter(services)
-
 	network := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
 		DefaultService: "overlay",
 	})
 
-	mockers := createMockers()
-
-	config := simulations.ServerConfig{
-		DefaultMockerID: "randomNodes",
-		// DefaultMockerID: "bootNet",
-		Mockers: mockers,
-	}
-
 	log.Info("starting simulation server on 0.0.0.0:8888...")
-	http.ListenAndServe(":8888", simulations.NewServer(network, config))
+	http.ListenAndServe(":8888", simulations.NewServer(network))
 }

swarm/network/stream/common_test.go

@@ -44,14 +44,20 @@ import (
 )
 
 var (
+	deliveries map[discover.NodeID]*Delivery
+	stores     map[discover.NodeID]storage.ChunkStore
+	toAddr     func(discover.NodeID) *network.BzzAddr
+	peerCount  func(discover.NodeID) int
 	adapter    = flag.String("adapter", "sim", "type of simulation: sim|socket|exec|docker")
-	loglevel = flag.Int("loglevel", 4, "verbosity of logs")
+	loglevel   = flag.Int("loglevel", 2, "verbosity of logs")
 )
 
 var (
 	defaultSkipCheck bool
 	waitPeerErrC     chan error
 	chunkSize        = 4096
+	registries       map[discover.NodeID]*TestRegistry
+	createStoreFunc  func(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, error)
 )
 
 var services = adapters.Services{
@@ -71,9 +77,14 @@ func init() {
 
 // NewStreamerService
 func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
+	var err error
 	id := ctx.Config.ID
 	addr := toAddr(id)
 	kad := network.NewKademlia(addr.Over(), network.NewKadParams())
+	stores[id], err = createStoreFunc(id, addr)
+	if err != nil {
+		return nil, err
+	}
 	store := stores[id].(*storage.LocalStore)
 	db := storage.NewDBAPI(store)
 	delivery := NewDelivery(kad, db)
@@ -87,7 +98,25 @@ func NewStreamerService(ctx *adapters.ServiceContext) (node.Service, error) {
 		waitPeerErrC <- waitForPeers(r, 1*time.Second, peerCount(id))
 	}()
 	dpa := storage.NewDPA(storage.NewNetStore(store, nil), storage.NewDPAParams())
-	return &TestRegistry{Registry: r, dpa: dpa}, nil
+	testRegistry := &TestRegistry{Registry: r, dpa: dpa}
+	registries[id] = testRegistry
+	return testRegistry, nil
+}
+
+func datadirsCleanup() {
+	for _, id := range ids {
+		os.RemoveAll(datadirs[id])
+	}
+}
+
+//local stores need to be cleaned up after the sim is done
+func localStoreCleanup() {
+	log.Info("Cleaning up...")
+	for _, id := range ids {
+		registries[id].Close()
+		stores[id].Close()
+	}
+	log.Info("Local store cleanup done")
 }
 
 func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *storage.LocalStore, func(), error) {
@@ -187,7 +216,7 @@ func (r *TestRegistry) APIs() []rpc.API {
 }
 
 func readAll(dpa *storage.DPA, hash []byte) (int64, error) {
-	r := dpa.Retrieve(hash)
+	r, _ := dpa.Retrieve(hash)
 	buf := make([]byte, 1024)
 	var n int
 	var total int64

swarm/network/stream/delivery.go

@@ -127,7 +127,7 @@ type RetrieveRequestMsg struct {
 }
 
 func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) error {
-	log.Debug("received request", "peer", sp.ID(), "hash", req.Key)
+	log.Trace("received request", "peer", sp.ID(), "hash", req.Key)
 	s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", false))
 	if err != nil {
 		return err
@@ -157,6 +157,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e
 			if req.SkipCheck {
 				err := sp.Deliver(chunk, s.priority)
 				if err != nil {
+					log.Warn("ERROR in handleRetrieveRequestMsg, DROPPING peer!", "err", err)
 					sp.Drop(err)
 				}
 			}

swarm/network/stream/delivery_test.go

@@ -22,6 +22,7 @@ import (
 	crand "crypto/rand"
 	"fmt"
 	"io"
+	"sync"
 	"testing"
 	"time"
 
@@ -36,13 +37,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-var (
-	deliveries map[discover.NodeID]*Delivery
-	stores     map[discover.NodeID]storage.ChunkStore
-	toAddr     func(discover.NodeID) *network.BzzAddr
-	peerCount  func(discover.NodeID) int
-)
-
 func TestStreamerRetrieveRequest(t *testing.T) {
 	tester, streamer, _, teardown, err := newStreamerTester(t)
 	defer teardown()
@@ -314,6 +308,7 @@ func TestDeliveryFromNodes(t *testing.T) {
 func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool) {
 	defaultSkipCheck = skipCheck
 	toAddr = network.NewAddrFromNodeID
+	createStoreFunc = createTestLocalStorageFromSim
 	conf := &streamTesting.RunConfig{
 		Adapter:         *adapter,
 		NodeCount:       nodes,
@@ -324,15 +319,20 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 	}
 
 	sim, teardown, err := streamTesting.NewSimulation(conf)
-	defer teardown()
+	var rpcSubscriptionsWg sync.WaitGroup
+	defer func() {
+		rpcSubscriptionsWg.Wait()
+		teardown()
+	}()
 	if err != nil {
 		t.Fatal(err.Error())
 	}
 	stores = make(map[discover.NodeID]storage.ChunkStore)
-	deliveries = make(map[discover.NodeID]*Delivery)
 	for i, id := range sim.IDs {
 		stores[id] = sim.Stores[i]
 	}
+	registries = make(map[discover.NodeID]*TestRegistry)
+	deliveries = make(map[discover.NodeID]*Delivery)
 	peerCount = func(id discover.NodeID) int {
 		if sim.IDs[0] == id || sim.IDs[nodes-1] == id {
 			return 1
@@ -352,6 +352,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 	errc := make(chan error, 1)
 	waitPeerErrC = make(chan error)
 	quitC := make(chan struct{})
+	defer close(quitC)
 
 	action := func(ctx context.Context) error {
 		// each node Subscribes to each other's swarmChunkServerStreamName
@@ -374,10 +375,15 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 		for j := 0; j < nodes-1; j++ {
 			id := sim.IDs[j]
 			err := sim.CallClient(id, func(client *rpc.Client) error {
-				err := streamTesting.WatchDisconnections(id, client, errc, quitC)
+				doneC, err := streamTesting.WatchDisconnections(id, client, errc, quitC)
 				if err != nil {
 					return err
 				}
+				rpcSubscriptionsWg.Add(1)
+				go func() {
+					<-doneC
+					rpcSubscriptionsWg.Done()
+				}()
 				ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
 				defer cancel()
 				sid := sim.IDs[j+1]
@@ -480,6 +486,8 @@ func BenchmarkDeliveryFromNodesWithCheck(b *testing.B) {
 func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skipCheck bool) {
 	defaultSkipCheck = skipCheck
 	toAddr = network.NewAddrFromNodeID
+	createStoreFunc = createTestLocalStorageFromSim
+	registries = make(map[discover.NodeID]*TestRegistry)
 
 	timeout := 300 * time.Second
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
@@ -494,7 +502,11 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
 		EnableMsgEvents: false,
 	}
 	sim, teardown, err := streamTesting.NewSimulation(conf)
-	defer teardown()
+	var rpcSubscriptionsWg sync.WaitGroup
+	defer func() {
+		rpcSubscriptionsWg.Wait()
+		teardown()
+	}()
 	if err != nil {
 		b.Fatal(err.Error())
 	}
@@ -544,10 +556,15 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, conns, chunkCount int, skip
 		for j := 0; j < nodes-1; j++ {
 			id := sim.IDs[j]
 			err = sim.CallClient(id, func(client *rpc.Client) error {
-				err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
+				doneC, err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
 				if err != nil {
 					return err
 				}
+				rpcSubscriptionsWg.Add(1)
+				go func() {
+					<-doneC
+					rpcSubscriptionsWg.Done()
+				}()
 				ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
 				defer cancel()
 				sid := sim.IDs[j+1] // the upstream peer's id
@@ -674,3 +691,7 @@ Loop:
 		b.Fatalf("expected no error. got %v", err)
 	}
 }
+
+func createTestLocalStorageFromSim(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, error) {
+	return stores[id], nil
+}

swarm/network/stream/intervals_test.go

@@ -22,6 +22,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
+	"sync"
 	"testing"
 	"time"
 
@@ -90,7 +91,11 @@ func testIntervals(t *testing.T, live bool, history *Range) {
 	}
 
 	sim, teardown, err := streamTesting.NewSimulation(conf)
-	defer teardown()
+	var rpcSubscriptionsWg sync.WaitGroup
+	defer func() {
+		rpcSubscriptionsWg.Wait()
+		teardown()
+	}()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -136,10 +141,15 @@ func testIntervals(t *testing.T, live bool, history *Range) {
 
 			sid := sim.IDs[0]
 
-			err := streamTesting.WatchDisconnections(id, client, errc, quitC)
+			doneC, err := streamTesting.WatchDisconnections(id, client, errc, quitC)
 			if err != nil {
 				return err
 			}
+			rpcSubscriptionsWg.Add(1)
+			go func() {
+				<-doneC
+				rpcSubscriptionsWg.Done()
+			}()
 			ctx, cancel := context.WithTimeout(ctx, 100*time.Second)
 			defer cancel()
 

swarm/network/stream/messages.go

@@ -85,7 +85,7 @@ func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) (err error) {
 		}
 	}()
 
-	log.Debug("received subscription", "peer", p.ID(), "stream", req.Stream, "history", req.History)
+	log.Debug("%s received subscription", "from", p.streamer.addr.ID(), "peer", p.ID(), "stream", req.Stream, "history", req.History)
 
 	f, err := p.streamer.GetServerFunc(req.Stream.Name)
 	if err != nil {
@@ -110,6 +110,7 @@ func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) (err error) {
 
 	go func() {
 		if err := p.SendOfferedHashes(os, from, to); err != nil {
+			log.Warn("ERROR in SendOfferedHashes, DROPPING peer!", "err", err)
 			p.Drop(err)
 		}
 	}()
@@ -127,6 +128,7 @@ func (p *Peer) handleSubscribeMsg(req *SubscribeMsg) (err error) {
 		}
 		go func() {
 			if err := p.SendOfferedHashes(os, req.History.From, req.History.To); err != nil {
+				log.Warn("ERROR in SendOfferedHashes, DROPPING peer!", "err", err)
 				p.Drop(err)
 			}
 		}()
@@ -235,11 +237,13 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error {
 	}
 	go func() {
 		select {
-		case <-time.After(30 * time.Second):
+		case <-time.After(120 * time.Second):
+			log.Warn("ERROR in handleOfferedHashesMsg, DROPPING peer!", "err", "TIMEOUT")
 			p.Drop(err)
 			return
 		case err := <-c.next:
 			if err != nil {
+				log.Warn("ERROR in handleOfferedHashesMsg, DROPPING peer!", "err", err)
 				p.Drop(err)
 				return
 			}
@@ -247,6 +251,7 @@ func (p *Peer) handleOfferedHashesMsg(req *OfferedHashesMsg) error {
 		log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "from", msg.From, "to", msg.To)
 		err := p.SendPriority(msg, c.priority)
 		if err != nil {
+			log.Warn("ERROR in handleOfferedHashesMsg, DROPPING peer!", "err", err)
 			p.Drop(err)
 		}
 	}()
@@ -279,6 +284,7 @@ func (p *Peer) handleWantedHashesMsg(req *WantedHashesMsg) error {
 	// launch in go routine since GetBatch blocks until new hashes arrive
 	go func() {
 		if err := p.SendOfferedHashes(s, req.From, req.To); err != nil {
+			log.Warn("ERROR in handleWantedHashesMsg, DROPPING peer!", "err", err)
 			p.Drop(err)
 		}
 	}()

swarm/network/stream/peer.go

@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/storage"
 )
 
-var sendTimeout = 5 * time.Second
+var sendTimeout = 30 * time.Second
 
 type notFoundError struct {
 	t string

swarm/network/stream/snapshot_sync_test.go

@@ -0,0 +1,670 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+package stream
+
+import (
+	"context"
+	crand "crypto/rand"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/p2p"
+	"github.com/ethereum/go-ethereum/p2p/discover"
+	"github.com/ethereum/go-ethereum/p2p/simulations"
+	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
+	"github.com/ethereum/go-ethereum/pot"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/network"
+	streamTesting "github.com/ethereum/go-ethereum/swarm/network/stream/testing"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+)
+
+const testMinProxBinSize = 2
+const MAX_TIMEOUT = 600
+
+var (
+	pof = pot.DefaultPof(256)
+
+	conf      *synctestConfig
+	startTime time.Time
+	ids       []discover.NodeID
+	datadirs  map[discover.NodeID]string
+	ppmap     map[string]*network.PeerPot
+
+	globalWg sync.WaitGroup
+
+	live    bool
+	history bool
+
+	longrunning = flag.Bool("longrunning", false, "do run long-running tests")
+)
+
+type synctestConfig struct {
+	addrs            [][]byte
+	chunks           []storage.Key
+	idToChunksMap    map[discover.NodeID][]int
+	chunksToNodesMap map[string][]int
+	idToAddrMap      map[discover.NodeID][]byte
+	addrToIdMap      map[string]discover.NodeID
+}
+
+func init() {
+	rand.Seed(time.Now().Unix())
+}
+
+//common_test needs to initialize the test in a init() func
+//in order for adapters to register the NewStreamerService;
+//this service is dependent on some global variables
+//we thus need to initialize first as init() as well.
+func initSyncTest() {
+	//assign the toAddr func so NewStreamerService can build the addr
+	toAddr = func(id discover.NodeID) *network.BzzAddr {
+		addr := network.NewAddrFromNodeID(id)
+		return addr
+	}
+
+	createStoreFunc = createTestLocalStorageForId
+	//local stores
+	stores = make(map[discover.NodeID]storage.ChunkStore)
+	//data directories for each node and store
+	datadirs = make(map[discover.NodeID]string)
+	//deliveries for each node
+	deliveries = make(map[discover.NodeID]*Delivery)
+	//registries, map of discover.NodeID to its streamer
+	registries = make(map[discover.NodeID]*TestRegistry)
+	//channel to wait for peers connected
+	//not needed for this test but required from common_test for NewStreamService
+	waitPeerErrC = make(chan error)
+	//also not needed for this test but required for NewStreamService
+	peerCount = func(id discover.NodeID) int {
+		if ids[0] == id || ids[len(ids)-1] == id {
+			return 1
+		}
+		return 2
+	}
+}
+
+//This file executes a number of tests with the syntax
+//TestSyncing_x_y
+//x is the number of chunks which will be uploaded
+//y is the number of nodes for the test
+func TestSyncing_4_32(t *testing.T)  { testSyncing(t, 4, 32) }
+func TestSyncing_32_16(t *testing.T) { testSyncing(t, 32, 16) }
+
+func TestLongRunningSyncing(t *testing.T) {
+	if *longrunning {
+		chnkCnt := []int{1, 8, 32, 256, 1024}
+		nCnt := []int{16, 32, 64, 128, 256}
+
+		for _, chnk := range chnkCnt {
+			for _, n := range nCnt {
+				log.Info(fmt.Sprintf("Long running test with %d chunks and %d nodes...", chnk, n))
+				testSyncing(t, chnk, n)
+			}
+		}
+	}
+}
+
+//do run the tests
+func testSyncing(t *testing.T, chunkCount int, nodeCount int) {
+	initSyncTest()
+	ids = make([]discover.NodeID, nodeCount)
+
+	//test live and NO history
+	log.Info("Testing live and no history")
+	live = true
+	history = false
+	err := runSyncTest(chunkCount, nodeCount, live, history)
+	if err != nil {
+		t.Fatal(err)
+	}
+	//test history only
+	log.Info("Testing history only")
+	live = false
+	history = true
+	err = runSyncTest(chunkCount, nodeCount, live, history)
+	if err != nil {
+		t.Fatal(err)
+	}
+	//finally test live and history
+	log.Info("Testing live and history")
+	live = true
+	err = runSyncTest(chunkCount, nodeCount, live, history)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+/*
+The test generates the given number of chunks,
+then uploads these to a random node.
+Afterwards for every chunk generated, the nearest node addresses
+are identified, syncing is started, and finally we verify
+that the nodes closer to the chunk addresses actually do have
+the chunks in their local stores.
+
+The test loads a snapshot file to construct the swarm network,
+assuming that the snapshot file identifies a healthy
+kademlia network. The snapshot should have 'streamer' in its service list.
+
+For every test run, a series of three tests will be executed:
+- a LIVE test first, where first subscriptions are established,
+  then a file (random chunks) is uploaded
+- a HISTORY test, where the file is uploaded first, and then
+  the subscriptions are established
+- a crude LIVE AND HISTORY test last, where (different) chunks
+  are uploaded twice, once before and once after subscriptions
+*/
+func runSyncTest(chunkCount int, nodeCount int, live bool, history bool) error {
+	//initialize the test struct
+	conf = &synctestConfig{}
+	//map of discover ID to indexes of chunks expected at that ID
+	conf.idToChunksMap = make(map[discover.NodeID][]int)
+	//map of discover ID to kademlia overlay address
+	conf.idToAddrMap = make(map[discover.NodeID][]byte)
+	//map of overlay address to discover ID
+	conf.addrToIdMap = make(map[string]discover.NodeID)
+	conf.chunks = make([]storage.Key, 0)
+	//First load the snapshot from the file
+	trigger := make(chan discover.NodeID)
+	// channel to signal simulation initialisation with action call complete
+	// or node disconnections
+	disconnectC := make(chan error)
+	quitC := make(chan struct{})
+
+	//load nodes from the snapshot file
+	net, err := initNetWithSnapshot(nodeCount)
+	if err != nil {
+		return err
+	}
+	var rpcSubscriptionsWg sync.WaitGroup
+	//do cleanup after test is terminated
+	defer func() {
+		// close quitC channel to signall all goroutines to clanup
+		// before calling simulation network shutdown.
+		close(quitC)
+		//wait for all rpc subscriptions to unsubscribe
+		rpcSubscriptionsWg.Wait()
+		//shutdown the snapshot network
+		net.Shutdown()
+		//after the test, clean up local stores initialized with createLocalStoreForId
+		localStoreCleanup()
+		//finally clear all data directories
+		datadirsCleanup()
+	}()
+	//get the nodes of the network
+	nodes := net.GetNodes()
+	//select one index at random...
+	idx := rand.Intn(len(nodes))
+	//...and get the the node at that index
+	//this is the node selected for upload
+	node := nodes[idx]
+
+	log.Info("Initializing test config")
+	//iterate over all nodes...
+	for c := 0; c < len(nodes); c++ {
+		//create an array of discovery node IDs
+		ids[c] = nodes[c].ID()
+		//get the kademlia overlay address from this ID
+		a := network.ToOverlayAddr(ids[c].Bytes())
+		//append it to the array of all overlay addresses
+		conf.addrs = append(conf.addrs, a)
+		//the proximity calculation is on overlay addr,
+		//the p2p/simulations check func triggers on discover.NodeID,
+		//so we need to know which overlay addr maps to which nodeID
+		conf.idToAddrMap[ids[c]] = a
+		conf.addrToIdMap[string(a)] = ids[c]
+	}
+	log.Info("Test config successfully initialized")
+
+	//only needed for healthy call when debugging
+	ppmap = network.NewPeerPotMap(testMinProxBinSize, conf.addrs)
+
+	//define the action to be performed before the test checks: start syncing
+	action := func(ctx context.Context) error {
+		ticker := time.NewTicker(200 * time.Millisecond)
+		defer ticker.Stop()
+		for range ticker.C {
+			healthy := true
+			for _, id := range ids {
+				r := registries[id]
+				//PeerPot for this node
+				addr := common.Bytes2Hex(network.ToOverlayAddr(id.Bytes()))
+				pp := ppmap[addr]
+				//call Healthy RPC
+				h := r.delivery.overlay.Healthy(pp)
+				//print info
+				log.Debug(r.delivery.overlay.String())
+				log.Debug(fmt.Sprintf("IS HEALTHY: %t", h.GotNN && h.KnowNN && h.Full))
+				if !h.GotNN || !h.Full {
+					healthy = false
+					break
+				}
+			}
+			if healthy {
+				break
+			}
+		}
+
+		if history {
+			log.Info("Uploading for history")
+			//If testing only history, we upload the chunk(s) first
+			chunks, err := uploadFileToSingleNodeStore(node.ID(), chunkCount)
+			if err != nil {
+				return err
+			}
+			conf.chunks = append(conf.chunks, chunks...)
+			//finally map chunks to the closest addresses
+			mapKeysToNodes(conf)
+		}
+
+		//variables needed to wait for all subscriptions established before uploading
+		errc := make(chan error)
+
+		//now setup and start event watching in order to know when we can upload
+		ctx, watchCancel := context.WithTimeout(context.Background(), MAX_TIMEOUT*time.Second)
+		defer watchCancel()
+
+		log.Info("Setting up stream subscription")
+		// each node Subscribes to each other's swarmChunkServerStreamName
+		for j, id := range ids {
+			log.Trace(fmt.Sprintf("subscribe: %d", j))
+			client, err := net.GetNode(id).Client()
+			if err != nil {
+				return err
+			}
+
+			wsDoneC := watchSubscriptionEvents(ctx, id, client, errc, quitC)
+			// doneC is nil, the error happened which is sent to errc channel, already
+			if wsDoneC == nil {
+				continue
+			}
+			rpcSubscriptionsWg.Add(1)
+			go func() {
+				<-wsDoneC
+				rpcSubscriptionsWg.Done()
+			}()
+
+			if log.Lvl(*loglevel) >= log.LvlTrace {
+				//this will print the kademlia tables of all nodes
+				//to only print the kademlia of the pivot node,
+				//use: if j == idx {}
+				var kt string
+				err = client.CallContext(ctx, &kt, "stream_getKad")
+				if err != nil {
+					return err
+				}
+
+				log.Debug("kad table " + node.ID().String())
+				log.Debug(kt)
+			}
+			//watch for peers disconnecting
+			wdDoneC, err := streamTesting.WatchDisconnections(id, client, disconnectC, quitC)
+			if err != nil {
+				return err
+			}
+			rpcSubscriptionsWg.Add(1)
+			go func() {
+				<-wdDoneC
+				rpcSubscriptionsWg.Done()
+			}()
+			//start syncing!
+			err = client.CallContext(ctx, nil, "stream_startSyncing")
+			if err != nil {
+				return err
+			}
+		}
+
+		//now wait until the number of expected subscriptions has been finished
+		go func() {
+			globalWg.Wait()
+			errc <- nil
+		}()
+
+		err := <-errc
+		if err != nil {
+			return err
+		}
+		log.Info("Stream subscriptions successfully requested")
+		if live {
+			//now upload the chunks to the selected random single node
+			chunks, err := uploadFileToSingleNodeStore(node.ID(), chunkCount)
+			if err != nil {
+				return err
+			}
+			conf.chunks = append(conf.chunks, chunks...)
+			//finally map chunks to the closest addresses
+			log.Debug(fmt.Sprintf("Uploaded chunks for live syncing: %v", conf.chunks))
+			mapKeysToNodes(conf)
+			log.Info(fmt.Sprintf("Uploaded %d chunks to random single node", chunkCount))
+		}
+
+		log.Info("Action terminated")
+
+		return nil
+	}
+
+	//check defines what will be checked during the test
+	check := func(ctx context.Context, id discover.NodeID) (bool, error) {
+		select {
+		case <-ctx.Done():
+			return false, ctx.Err()
+		case e := <-disconnectC:
+			log.Error(e.Error())
+			return false, fmt.Errorf("Disconnect event detected, network unhealthy")
+		default:
+		}
+		log.Trace(fmt.Sprintf("Checking node: %s", id))
+		//select the local store for the given node
+		lstore := stores[id]
+		//if there are more than one chunk, test only succeeds if all expected chunks are found
+		allSuccess := true
+
+		//all the chunk indexes which are supposed to be found for this node
+		localChunks := conf.idToChunksMap[id]
+		//for each expected chunk, check if it is in the local store
+		for _, ch := range localChunks {
+			//get the real chunk by the index in the index array
+			chunk := conf.chunks[ch]
+			log.Trace(fmt.Sprintf("node has chunk: %s:", chunk))
+			//check if the expected chunk is indeed in the localstore
+			if _, err := lstore.Get(chunk); err != nil {
+				log.Warn(fmt.Sprintf("Chunk %s NOT found for id %s", chunk, id))
+				allSuccess = false
+			} else {
+				log.Debug(fmt.Sprintf("Chunk %s IS FOUND for id %s", chunk, id))
+			}
+		}
+
+		return allSuccess, nil
+	}
+
+	//for each tick, run the checks on all nodes
+	timingTicker := time.NewTicker(time.Second * 1)
+	defer timingTicker.Stop()
+	go func() {
+		for range timingTicker.C {
+			for i := 0; i < len(ids); i++ {
+				log.Trace(fmt.Sprintf("triggering step %d, id %s", i, ids[i]))
+				trigger <- ids[i]
+			}
+		}
+	}()
+
+	log.Info("Starting simulation run...")
+
+	timeout := MAX_TIMEOUT * time.Second
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	//run the simulation
+	result := simulations.NewSimulation(net).Run(ctx, &simulations.Step{
+		Action:  action,
+		Trigger: trigger,
+		Expect: &simulations.Expectation{
+			Nodes: ids,
+			Check: check,
+		},
+	})
+
+	if result.Error != nil {
+		return result.Error
+	}
+	log.Info("Simulation terminated")
+	return nil
+}
+
+//Show kademlia of uploading node for debugging
+func (r *TestRegistry) GetKad(ctx context.Context) string {
+	return r.delivery.overlay.String()
+}
+
+//the server func to start syncing
+func (r *TestRegistry) StartSyncing(ctx context.Context) error {
+	var err error
+
+	if log.Lvl(*loglevel) == log.LvlDebug {
+		//PeerPot for this node
+		addr := common.Bytes2Hex(r.addr.OAddr)
+		pp := ppmap[addr]
+		//call Healthy RPC
+		h := r.delivery.overlay.Healthy(pp)
+		//print info
+		log.Debug(r.delivery.overlay.String())
+		log.Debug(fmt.Sprintf("IS HEALTHY: %t", h.GotNN && h.KnowNN && h.Full))
+	}
+
+	kad, ok := r.delivery.overlay.(*network.Kademlia)
+	if !ok {
+		return fmt.Errorf("Not a Kademlia!")
+	}
+
+	//iterate over each bin and solicit needed subscription to bins
+	kad.EachBin(r.addr.Over(), pof, 0, func(conn network.OverlayConn, po int) bool {
+		//identify begin and start index of the bin(s) we want to subscribe to
+		log.Debug(fmt.Sprintf("Requesting subscription by: registry %s from peer %s for bin: %d", r.addr.ID(), conf.addrToIdMap[string(conn.Address())], po))
+		var histRange *Range
+		if history {
+			histRange = &Range{}
+		}
+
+		globalWg.Add(1)
+		err = r.RequestSubscription(conf.addrToIdMap[string(conn.Address())], NewStream("SYNC", FormatSyncBinKey(uint8(po)), live), histRange, Top)
+		if err != nil {
+			log.Error(fmt.Sprintf("Error in RequestSubsciption! %v", err))
+			return false
+		}
+		return true
+
+	})
+	return nil
+}
+
+//map chunk keys to addresses which are responsible
+func mapKeysToNodes(conf *synctestConfig) {
+	kmap := make(map[string][]int)
+	nodemap := make(map[string][]int)
+	//build a pot for chunk hashes
+	np := pot.NewPot(nil, 0)
+	indexmap := make(map[string]int)
+	for i, a := range conf.addrs {
+		indexmap[string(a)] = i
+		np, _, _ = pot.Add(np, a, pof)
+	}
+	//for each address, run EachNeighbour on the chunk hashes pot to identify closest nodes
+	log.Trace(fmt.Sprintf("Generated hash chunk(s): %v", conf.chunks))
+	for i := 0; i < len(conf.chunks); i++ {
+		pl := 256 //highest possible proximity
+		var nns []int
+		np.EachNeighbour([]byte(conf.chunks[i]), pof, func(val pot.Val, po int) bool {
+			a := val.([]byte)
+			if pl < 256 && pl != po {
+				return false
+			}
+			if pl == 256 || pl == po {
+				log.Trace(fmt.Sprintf("appending %s", conf.addrToIdMap[string(a)]))
+				nns = append(nns, indexmap[string(a)])
+				nodemap[string(a)] = append(nodemap[string(a)], i)
+			}
+			if pl == 256 && len(nns) >= testMinProxBinSize {
+				//maxProxBinSize has been reached at this po, so save it
+				//we will add all other nodes at the same po
+				pl = po
+			}
+			return true
+		})
+		kmap[string(conf.chunks[i])] = nns
+	}
+	for addr, chunks := range nodemap {
+		//this selects which chunks are expected to be found with the given node
+		conf.idToChunksMap[conf.addrToIdMap[addr]] = chunks
+	}
+	log.Debug(fmt.Sprintf("Map of expected chunks by ID: %v", conf.idToChunksMap))
+	conf.chunksToNodesMap = kmap
+}
+
+//upload a file(chunks) to a single local node store
+func uploadFileToSingleNodeStore(id discover.NodeID, chunkCount int) ([]storage.Key, error) {
+	log.Debug(fmt.Sprintf("Uploading to node id: %s", id))
+	lstore := stores[id]
+	size := chunkSize
+	dpa := storage.NewDPA(lstore, storage.NewDPAParams())
+	var rootkeys []storage.Key
+	for i := 0; i < chunkCount; i++ {
+		rk, wait, err := dpa.Store(io.LimitReader(crand.Reader, int64(size)), int64(size), false)
+		wait()
+		if err != nil {
+			return nil, err
+		}
+		rootkeys = append(rootkeys, (rk))
+	}
+
+	return rootkeys, nil
+}
+
+//initialize a network from a snapshot
+func initNetWithSnapshot(nodeCount int) (*simulations.Network, error) {
+
+	var a adapters.NodeAdapter
+	//add the streamer service to the node adapter
+
+	if *adapter == "exec" {
+		dirname, err := ioutil.TempDir(".", "")
+		if err != nil {
+			return nil, err
+		}
+		a = adapters.NewExecAdapter(dirname)
+	} else if *adapter == "socket" {
+		a = adapters.NewSocketAdapter(services)
+	} else if *adapter == "tcp" {
+		a = adapters.NewTCPAdapter(services)
+	} else if *adapter == "sim" {
+		a = adapters.NewSimAdapter(services)
+	}
+
+	log.Info("Setting up Snapshot network")
+
+	net := simulations.NewNetwork(a, &simulations.NetworkConfig{
+		ID:             "0",
+		DefaultService: "streamer",
+	})
+
+	f, err := os.Open(fmt.Sprintf("testing/snapshot_%d.json", nodeCount))
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	jsonbyte, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	var snap simulations.Snapshot
+	err = json.Unmarshal(jsonbyte, &snap)
+	if err != nil {
+		return nil, err
+	}
+
+	//the snapshot probably has the property EnableMsgEvents not set
+	//just in case, set it to true!
+	//(we need this to wait for messages before uploading)
+	for _, n := range snap.Nodes {
+		n.Node.Config.EnableMsgEvents = true
+	}
+
+	log.Info("Waiting for p2p connections to be established...")
+
+	//now we can load the snapshot
+	err = net.Load(&snap)
+	if err != nil {
+		return nil, err
+	}
+	log.Info("Snapshot loaded")
+	return net, nil
+}
+
+//we want to wait for subscriptions to be established before uploading to test
+//that live syncing is working correctly
+func watchSubscriptionEvents(ctx context.Context, id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) (doneC <-chan struct{}) {
+	events := make(chan *p2p.PeerEvent)
+	sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
+	if err != nil {
+		log.Error(err.Error())
+		errc <- fmt.Errorf("error getting peer events for node %v: %s", id, err)
+		return
+	}
+	c := make(chan struct{})
+	go func() {
+		defer func() {
+			log.Trace("watch subscription events: unsubscribe", "id", id)
+			sub.Unsubscribe()
+			close(c)
+		}()
+
+		for {
+			select {
+			case <-quitC:
+				return
+			case <-ctx.Done():
+				select {
+				case errc <- ctx.Err():
+				case <-quitC:
+				}
+				return
+			case e := <-events:
+				//just catch SubscribeMsg
+				if e.Type == p2p.PeerEventTypeMsgRecv && e.Protocol == "stream" && e.MsgCode != nil && *e.MsgCode == 4 {
+					globalWg.Done()
+				}
+			case err := <-sub.Err():
+				if err != nil {
+					select {
+					case errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err):
+					case <-quitC:
+					}
+					return
+				}
+			}
+		}
+	}()
+	return c
+}
+
+//create a local store for the given node
+func createTestLocalStorageForId(id discover.NodeID, addr *network.BzzAddr) (storage.ChunkStore, error) {
+	var datadir string
+	var err error
+	datadir, err = ioutil.TempDir("", fmt.Sprintf("syncer-test-%s", id.TerminalString()))
+	if err != nil {
+		return nil, err
+	}
+	datadirs[id] = datadir
+	var store storage.ChunkStore
+	store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+	if err != nil {
+		return nil, err
+	}
+	return store, nil
+}

swarm/network/stream/stream.go

@@ -93,7 +93,7 @@ func NewRegistry(addr *network.BzzAddr, delivery *Delivery, db *storage.DBAPI, i
 		return NewSwarmChunkServer(delivery.db), nil
 	})
 	streamer.RegisterClientFunc(swarmChunkServerStreamName, func(p *Peer, _ string, _ bool) (Client, error) {
-		return NewSwarmSyncerClient(p, delivery.db)
+		return NewSwarmSyncerClient(p, delivery.db, false)
 	})
 	RegisterSwarmSyncerServer(streamer, db)
 	RegisterSwarmSyncerClient(streamer, db)

swarm/network/stream/streamer_test.go

@@ -39,6 +39,20 @@ func TestStreamerSubscribe(t *testing.T) {
 	}
 }
 
+func TestStreamerRequestSubscription(t *testing.T) {
+	tester, streamer, _, teardown, err := newStreamerTester(t)
+	defer teardown()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stream := NewStream("foo", "", false)
+	err = streamer.RequestSubscription(tester.IDs[0], stream, &Range{}, Top)
+	if err == nil || err.Error() != "stream foo not registered" {
+		t.Fatalf("Expected error %v, got %v", "stream foo not registered", err)
+	}
+}
+
 var (
 	hash0     = sha3.Sum256([]byte{0})
 	hash1     = sha3.Sum256([]byte{1})

swarm/network/stream/syncer.go

@@ -138,12 +138,14 @@ type SwarmSyncerClient struct {
 	currentRoot           storage.Key
 	requestFunc           func(chunk *storage.Chunk)
 	end, start            uint64
+	ignoreExistingRequest bool
 }
 
 // NewSwarmSyncerClient is a contructor for provable data exchange syncer
-func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI) (*SwarmSyncerClient, error) {
+func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI, ignoreExistingRequest bool) (*SwarmSyncerClient, error) {
 	return &SwarmSyncerClient{
 		db: db,
+		ignoreExistingRequest: ignoreExistingRequest,
 	}, nil
 }
 
@@ -187,15 +189,15 @@ func NewSwarmSyncerClient(_ *Peer, db *storage.DBAPI) (*SwarmSyncerClient, error
 // to handle incoming sync streams
 func RegisterSwarmSyncerClient(streamer *Registry, db *storage.DBAPI) {
 	streamer.RegisterClientFunc("SYNC", func(p *Peer, _ string, love bool) (Client, error) {
-		return NewSwarmSyncerClient(p, db)
+		return NewSwarmSyncerClient(p, db, true)
 	})
 }
 
 // NeedData
 func (s *SwarmSyncerClient) NeedData(key []byte) (wait func()) {
-	chunk, _ := s.db.GetOrCreateRequest(key)
+	chunk, created := s.db.GetOrCreateRequest(key)
 	// TODO: we may want to request from this peer anyway even if the request exists
-	if chunk.ReqC == nil {
+	if chunk.ReqC == nil || (s.ignoreExistingRequest && !created) {
 		return nil
 	}
 	// create request and wait until the chunk data arrives and is stored

swarm/network/stream/syncer_test.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"sync"
 	"testing"
 	"time"
 
@@ -45,8 +46,11 @@ func TestSyncerSimulation(t *testing.T) {
 
 func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck bool, po uint8) {
 	defaultSkipCheck = skipCheck
+	createStoreFunc = createTestLocalStorageFromSim
+	registries = make(map[discover.NodeID]*TestRegistry)
 	toAddr = func(id discover.NodeID) *network.BzzAddr {
 		addr := network.NewAddrFromNodeID(id)
+		//hack to put addresses in same space
 		addr.OAddr[0] = byte(0)
 		return addr
 	}
@@ -66,7 +70,11 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 
 	// create simulation network with the config
 	sim, teardown, err := streamTesting.NewSimulation(conf)
-	defer teardown()
+	var rpcSubscriptionsWg sync.WaitGroup
+	defer func() {
+		rpcSubscriptionsWg.Wait()
+		teardown()
+	}()
 	if err != nil {
 		t.Fatal(err.Error())
 	}
@@ -151,10 +159,15 @@ func testSyncBetweenNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck
 			id := sim.IDs[j]
 			err := sim.CallClient(id, func(client *rpc.Client) error {
 				// report disconnect events to the error channel cos peers should not disconnect
-				err := streamTesting.WatchDisconnections(id, client, errc, quitC)
+				doneC, err := streamTesting.WatchDisconnections(id, client, errc, quitC)
 				if err != nil {
 					return err
 				}
+				rpcSubscriptionsWg.Add(1)
+				go func() {
+					<-doneC
+					rpcSubscriptionsWg.Done()
+				}()
 				ctx, cancel := context.WithTimeout(ctx, 1*time.Second)
 				defer cancel()
 				// start syncing, i.e., subscribe to upstream peers po 1 bin

swarm/network/stream/testing/testing.go

@@ -216,27 +216,47 @@ func (s *Simulation) Run(ctx context.Context, conf *RunConfig) (*simulations.Ste
 	return result, nil
 }
 
-func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) error {
+// WatchDisconnections subscribes to admin peerEvents and sends peer event drop
+// errors to the errc channel. Channel quitC signals the termination of the event loop.
+// Returned doneC will be closed after the rpc subscription is unsubscribed,
+// signaling that simulations network is safe to shutdown.
+func WatchDisconnections(id discover.NodeID, client *rpc.Client, errc chan error, quitC chan struct{}) (doneC <-chan struct{}, err error) {
 	events := make(chan *p2p.PeerEvent)
 	sub, err := client.Subscribe(context.Background(), "admin", events, "peerEvents")
 	if err != nil {
-		return fmt.Errorf("error getting peer events for node %v: %s", id, err)
+		return nil, fmt.Errorf("error getting peer events for node %v: %s", id, err)
 	}
+	c := make(chan struct{})
 	go func() {
+		defer func() {
+			log.Trace("watch disconnections: unsubscribe", "id", id)
+			sub.Unsubscribe()
+			close(c)
+		}()
 		for {
 			select {
 			case <-quitC:
 				return
 			case e := <-events:
-				errc <- fmt.Errorf("peerEvent for node %v: %v", id, e)
+				if e.Type == p2p.PeerEventTypeDrop {
+					select {
+					case errc <- fmt.Errorf("peerEvent for node %v: %v", id, e):
+					case <-quitC:
+						return
+					}
+				}
 			case err := <-sub.Err():
 				if err != nil {
-					errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err)
+					select {
+					case errc <- fmt.Errorf("error getting peer events for node %v: %v", id, err):
+					case <-quitC:
+						return
+					}
 				}
 			}
 		}
 	}()
-	return nil
+	return c, nil
 }
 
 func Trigger(d time.Duration, quitC chan struct{}, ids ...discover.NodeID) chan discover.NodeID {

swarm/pss/api.go

@@ -2,6 +2,7 @@ package pss
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -122,7 +123,11 @@ func (pssapi *API) GetAsymmetricAddressHint(topic Topic, pubkeyid string) (PssAd
 }
 
 func (pssapi *API) StringToTopic(topicstring string) (Topic, error) {
-	return BytesToTopic([]byte(topicstring)), nil
+	topicbytes := BytesToTopic([]byte(topicstring))
+	if topicbytes == rawTopic {
+		return rawTopic, errors.New("Topic string hashes to 0x00000000 and cannot be used")
+	}
+	return topicbytes, nil
 }
 
 func (pssapi *API) SendAsym(pubkeyhex string, topic Topic, msg hexutil.Bytes) error {

swarm/pss/client/client_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"sync"
@@ -22,7 +21,6 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/state"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
 )
 
@@ -231,21 +229,13 @@ func newServices() adapters.Services {
 	}
 	return adapters.Services{
 		"pss": func(ctx *adapters.ServiceContext) (node.Service, error) {
-			cachedir, err := ioutil.TempDir("", "pss-cache")
-			if err != nil {
-				return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err)
-			}
-			dpa, err := storage.NewLocalDPA(cachedir, make([]byte, 32))
-			if err != nil {
-				return nil, fmt.Errorf("local dpa creation failed: %s", err)
-			}
 			ctxlocal, cancel := context.WithTimeout(context.Background(), time.Second)
 			defer cancel()
 			keys, err := wapi.NewKeyPair(ctxlocal)
 			privkey, err := w.GetPrivateKey(keys)
 			psparams := pss.NewPssParams(privkey)
 			pskad := kademlia(ctx.Config.ID)
-			ps := pss.NewPss(pskad, dpa, psparams)
+			ps := pss.NewPss(pskad, psparams)
 			pshparams := pss.NewHandshakeParams()
 			pshparams.SymKeySendLimit = sendLimit
 			err = pss.SetHandshakeController(ps, pshparams)

swarm/pss/protocol_test.go

@@ -37,7 +37,7 @@ func testProtocol(t *testing.T) {
 
 	topic := PingTopic.String()
 
-	clients, err := setupNetwork(2)
+	clients, err := setupNetwork(2, false)
 	if err != nil {
 		t.Fatal(err)
 	}

swarm/pss/pss.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"crypto/ecdsa"
 	"crypto/rand"
+	"errors"
 	"fmt"
 	"sync"
 	"time"
@@ -32,9 +33,10 @@ const (
 	defaultMaxMsgSize          = 1024 * 1024
 	defaultCleanInterval       = time.Second * 60 * 10
 	defaultDequeueInterval     = time.Millisecond * 10
-	defaultOutboxQueueSize     = 10000
+	defaultOutboxCapacity      = 10000
 	pssProtocolName            = "pss"
 	pssVersion                 = 1
+	hasherCount                = 8
 )
 
 var (
@@ -46,7 +48,6 @@ var (
 // and mailbox implementation
 type pssCacheEntry struct {
 	expiresAt time.Time
-	receivedFrom []byte
 }
 
 // abstraction to enable access to p2p.protocols.Peer.Send
@@ -71,6 +72,7 @@ type PssParams struct {
 	CacheTTL            time.Duration
 	privateKey          *ecdsa.PrivateKey
 	SymKeyCacheCapacity int
+	AllowRaw            bool // If true, enables sending and receiving messages without builtin pss encryption
 }
 
 // Sane defaults for Pss
@@ -89,7 +91,6 @@ func NewPssParams(privatekey *ecdsa.PrivateKey) *PssParams {
 type Pss struct {
 	network.Overlay                   // we can get the overlayaddress from this
 	privateKey      *ecdsa.PrivateKey // pss can have it's own independent key
-	dpa             *storage.DPA      // we use swarm to store the cache
 	w               *whisper.Whisper  // key and encryption backend
 	auxAPIs         []rpc.API         // builtins (handshake, test) can add APIs
 
@@ -116,6 +117,8 @@ type Pss struct {
 	// message handling
 	handlers   map[Topic]map[*Handler]bool // topic and version based pss payload handlers. See pss.Handle()
 	handlersMu sync.RWMutex
+	allowRaw   bool
+	hashPool   sync.Pool
 
 	// process
 	quitC chan struct{}
@@ -129,15 +132,14 @@ func (self *Pss) String() string {
 //
 // In addition to params, it takes a swarm network overlay
 // and a DPA storage for message cache storage.
-func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
+func NewPss(k network.Overlay, params *PssParams) *Pss {
 	cap := p2p.Cap{
 		Name:    pssProtocolName,
 		Version: pssVersion,
 	}
-	return &Pss{
+	ps := &Pss{
 		Overlay:    k,
 		privateKey: params.privateKey,
-		dpa:        dpa,
 		w:          whisper.New(&whisper.DefaultConfig),
 		quitC:      make(chan struct{}),
 
@@ -147,7 +149,7 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
 		msgTTL:          params.MsgTTL,
 		paddingByteSize: defaultPaddingByteSize,
 		capstring:       cap.String(),
-		outbox:          make(chan *PssMsg, defaultOutboxQueueSize),
+		outbox:          make(chan *PssMsg, defaultOutboxCapacity),
 
 		pubKeyPool:                 make(map[string]map[Topic]*pssPeer),
 		symKeyPool:                 make(map[string]map[Topic]*pssPeer),
@@ -155,7 +157,20 @@ func NewPss(k network.Overlay, dpa *storage.DPA, params *PssParams) *Pss {
 		symKeyDecryptCacheCapacity: params.SymKeyCacheCapacity,
 
 		handlers: make(map[Topic]map[*Handler]bool),
+		allowRaw: params.AllowRaw,
+		hashPool: sync.Pool{
+			New: func() interface{} {
+				return storage.MakeHashFunc(storage.SHA3Hash)()
+			},
+		},
 	}
+
+	for i := 0; i < hasherCount; i++ {
+		hashfunc := storage.MakeHashFunc(storage.SHA3Hash)()
+		ps.hashPool.Put(hashfunc)
+	}
+
+	return ps
 }
 
 /////////////////////////////////////////////////////////////////////
@@ -166,11 +181,14 @@ func (self *Pss) Start(srv *p2p.Server) error {
 	go func() {
 		for {
 			tickC := time.Tick(defaultCleanInterval)
+			cacheTickC := time.Tick(self.cacheTTL)
 			select {
+			case <-cacheTickC:
+				self.cleanFwdCache()
 			case <-tickC:
 				self.cleanKeys()
 			case <-self.quitC:
-				log.Info("pss shutting down")
+				return
 			}
 		}
 	}()
@@ -180,7 +198,7 @@ func (self *Pss) Start(srv *p2p.Server) error {
 			case msg := <-self.outbox:
 				self.forward(msg)
 			case <-self.quitC:
-				log.Info("pss shutting down")
+				return
 			}
 		}
 	}()
@@ -189,6 +207,7 @@ func (self *Pss) Start(srv *p2p.Server) error {
 }
 
 func (self *Pss) Stop() error {
+	log.Info("pss shutting down")
 	close(self.quitC)
 	return nil
 }
@@ -298,16 +317,28 @@ func (self *Pss) getHandlers(topic Topic) map[*Handler]bool {
 // Passes error to pss protocol handler if payload is not valid pssmsg
 func (self *Pss) handlePssMsg(msg interface{}) error {
 	pssmsg, ok := msg.(*PssMsg)
+
 	if ok {
+		if self.checkFwdCache(pssmsg) {
+			log.Trace(fmt.Sprintf("pss relay block-cache match (process): FROM %x TO %x", self.Overlay.BaseAddr(), common.ToHex(pssmsg.To)))
+			return nil
+		}
+		self.addFwdCache(pssmsg)
+
 		var err error
 		if !self.isSelfPossibleRecipient(pssmsg) {
 			log.Trace("pss was for someone else :'( ... forwarding", "pss", common.ToHex(self.BaseAddr()))
-			self.outbox <- pssmsg
+			if err := self.enqueue(pssmsg); err != nil {
+				return err
+			}
 		}
 		log.Trace("pss for us, yay! ... let's process!", "pss", common.ToHex(self.BaseAddr()))
 
-		if !self.process(pssmsg) {
+		if err := self.process(pssmsg); err != nil {
-			self.outbox <- pssmsg
+			qerr := self.enqueue(pssmsg)
+			if qerr != nil {
+				err = fmt.Errorf("%s + %s", err, qerr)
+			}
 		}
 		return err
 	}
@@ -318,7 +349,7 @@ func (self *Pss) handlePssMsg(msg interface{}) error {
 // Entry point to processing a message for which the current node can be the intended recipient.
 // Attempts symmetric and asymmetric decryption with stored keys.
 // Dispatches message to all handlers matching the message topic
-func (self *Pss) process(pssmsg *PssMsg) bool {
+func (self *Pss) process(pssmsg *PssMsg) error {
 	var err error
 	var recvmsg *whisper.ReceivedMessage
 	var from *PssAddress
@@ -328,6 +359,10 @@ func (self *Pss) process(pssmsg *PssMsg) bool {
 
 	envelope := pssmsg.Payload
 	psstopic := Topic(envelope.Topic)
+	if self.allowRaw && psstopic == rawTopic {
+		self.executeHandlers(rawTopic, envelope.Data, nil, false, "")
+		return nil
+	}
 
 	if len(envelope.AESNonce) > 0 { // detect symkey msg according to whisperv5/envelope.go:OpenSymmetric
 		keyFunc = self.processSym
@@ -337,26 +372,30 @@ func (self *Pss) process(pssmsg *PssMsg) bool {
 	}
 	recvmsg, keyid, from, err = keyFunc(envelope)
 	if err != nil {
-		log.Debug("decrypt message fail", "err", err, "asym", asymmetric, "pss", common.ToHex(self.BaseAddr()))
+		return errors.New("Decryption failed")
-		return false
 	}
 
 	if len(pssmsg.To) < addressLength {
-		go func() {
+		if err := self.enqueue(pssmsg); err != nil {
-			self.outbox <- pssmsg
+			return err
-		}()
 		}
-	handlers := self.getHandlers(psstopic)
+	}
+	self.executeHandlers(psstopic, recvmsg.Payload, from, asymmetric, keyid)
+
+	return nil
+
+}
+
+func (self *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, asymmetric bool, keyid string) {
+	handlers := self.getHandlers(topic)
 	nid, _ := discover.HexID("0x00") // this hack is needed to satisfy the p2p method
 	p := p2p.NewPeer(nid, fmt.Sprintf("%x", from), []p2p.Cap{})
 	for f := range handlers {
-		err := (*f)(recvmsg.Payload, p, asymmetric, keyid)
+		err := (*f)(payload, p, asymmetric, keyid)
 		if err != nil {
 			log.Warn("Pss handler %p failed: %v", f, err)
 		}
 	}
-	return true
-
 }
 
 // will return false if using partial address
@@ -532,7 +571,7 @@ func (self *Pss) cleanKeys() (count int) {
 	for keyid, peertopics := range self.symKeyPool {
 		var expiredtopics []Topic
 		for topic, psp := range peertopics {
-			log.Trace("check topic", "topic", topic, "id", keyid, "protect", psp.protected, "p", fmt.Sprintf("%p", self.symKeyPool[keyid][topic]))
+			//log.Trace("check topic", "topic", topic, "id", keyid, "protect", psp.protected, "p", fmt.Sprintf("%p", self.symKeyPool[keyid][topic]))
 			if psp.protected {
 				continue
 			}
@@ -540,7 +579,7 @@ func (self *Pss) cleanKeys() (count int) {
 			var match bool
 			for i := self.symKeyDecryptCacheCursor; i > self.symKeyDecryptCacheCursor-cap(self.symKeyDecryptCache) && i > 0; i-- {
 				cacheid := self.symKeyDecryptCache[i%cap(self.symKeyDecryptCache)]
-				log.Trace("check cache", "idx", i, "id", *cacheid)
+				//log.Trace("check cache", "idx", i, "id", *cacheid)
 				if *cacheid == keyid {
 					match = true
 				}
@@ -564,6 +603,35 @@ func (self *Pss) cleanKeys() (count int) {
 // SECTION: Message sending
 /////////////////////////////////////////////////////////////////////
 
+func (self *Pss) enqueue(msg *PssMsg) error {
+	select {
+	case self.outbox <- msg:
+		return nil
+	default:
+	}
+
+	return errors.New("outbox full")
+}
+
+// Send a raw message (any encryption is responsibility of calling client)
+//
+// Will fail if raw messages are disallowed
+func (self *Pss) SendRaw(msg []byte, address PssAddress) error {
+	if !self.allowRaw {
+		return errors.New("Raw messages not enabled")
+	}
+	pssmsg := &PssMsg{
+		To:     address,
+		Expire: uint32(time.Now().Add(self.msgTTL).Unix()),
+		Payload: &whisper.Envelope{
+			Data:  msg,
+			Topic: whisper.TopicType(rawTopic),
+		},
+	}
+	self.addFwdCache(pssmsg)
+	return self.enqueue(pssmsg)
+}
+
 // Send a message using symmetric encryption
 //
 // Fails if the key id does not match any of the stored symmetric keys
@@ -654,27 +722,19 @@ func (self *Pss) send(to []byte, topic Topic, msg []byte, asymmetric bool, key [
 		Expire:  uint32(time.Now().Add(self.msgTTL).Unix()),
 		Payload: envelope,
 	}
-	self.outbox <- pssmsg
+	return self.enqueue(pssmsg)
-	return nil
 }
 
 // Forwards a pss message to the peer(s) closest to the to recipient address in the PssMsg struct
 // The recipient address can be of any length, and the byte slice will be matched to the MSB slice
 // of the peer address of the equivalent length.
-func (self *Pss) forward(msg *PssMsg) {
+func (self *Pss) forward(msg *PssMsg) error {
 	to := make([]byte, addressLength)
 	copy(to[:len(msg.To)], msg.To)
 
-	// message hash
-	digest, err := self.storeMsg(msg)
-	if err != nil {
-		log.Warn(fmt.Sprintf("could not store message %v to cache: %v", msg, err))
-	}
-
 	// send with kademlia
 	// find the closest peer to the recipient and attempt to send
 	sent := 0
-
 	self.Overlay.EachConn(to, 256, func(op network.OverlayConn, po int, isproxbin bool) bool {
 		// we need p2p.protocols.Peer.Send
 		// cast and resolve
@@ -699,23 +759,19 @@ func (self *Pss) forward(msg *PssMsg) {
 		}
 
 		// get the protocol peer from the forwarding peer cache
-		sendMsg := fmt.Sprintf("MSG %x TO %x FROM %x VIA %x", digest, to, self.BaseAddr(), op.Address())
+		sendMsg := fmt.Sprintf("MSG TO %x FROM %x VIA %x", to, self.BaseAddr(), op.Address())
 		self.fwdPoolMu.RLock()
 		pp := self.fwdPool[sp.Info().ID]
 		self.fwdPoolMu.RUnlock()
-		if self.checkFwdCache(op.Address(), digest) {
+
-			log.Trace(fmt.Sprintf("%v: peer already forwarded to", sendMsg))
-			return true
-		}
 		// attempt to send the message
-		go func() {
 		err := pp.Send(msg)
 		if err != nil {
-				log.Debug(fmt.Sprintf("%v: failed forwarding: %v", sendMsg, err))
+			return true
 		}
-		}()
-		log.Trace(fmt.Sprintf("%v: successfully forwarded", sendMsg))
 		sent++
+		log.Trace(fmt.Sprintf("%v: successfully forwarded", sendMsg))
+
 		// continue forwarding if:
 		// - if the peer is end recipient but the full address has not been disclosed
 		// - if the peer address matches the partial address fully
@@ -737,23 +793,40 @@ func (self *Pss) forward(msg *PssMsg) {
 	if sent == 0 {
 		log.Debug("unable to forward to any peers")
 		time.Sleep(time.Millisecond)
-		self.outbox <- msg
+		if err := self.enqueue(msg); err != nil {
+			return err
+		}
 	}
 
 	// cache the message
-	self.addFwdCache(digest)
+	self.addFwdCache(msg)
+	return nil
 }
 
 /////////////////////////////////////////////////////////////////////
 // SECTION: Caching
 /////////////////////////////////////////////////////////////////////
 
-// add a message to the cache
+// cleanFwdCache is used to periodically remove expired entries from the forward cache
-func (self *Pss) addFwdCache(digest pssDigest) error {
+func (self *Pss) cleanFwdCache() {
 	self.fwdCacheMu.Lock()
 	defer self.fwdCacheMu.Unlock()
+	for k, v := range self.fwdCache {
+		if v.expiresAt.Before(time.Now()) {
+			delete(self.fwdCache, k)
+		}
+	}
+}
+
+// add a message to the cache
+func (self *Pss) addFwdCache(msg *PssMsg) error {
 	var entry pssCacheEntry
 	var ok bool
+
+	self.fwdCacheMu.Lock()
+	defer self.fwdCacheMu.Unlock()
+
+	digest := self.digest(msg)
 	if entry, ok = self.fwdCache[digest]; !ok {
 		entry = pssCacheEntry{}
 	}
@@ -763,34 +836,31 @@ func (self *Pss) addFwdCache(digest pssDigest) error {
 }
 
 // check if message is in the cache
-func (self *Pss) checkFwdCache(addr []byte, digest pssDigest) bool {
+func (self *Pss) checkFwdCache(msg *PssMsg) bool {
-	self.fwdCacheMu.RLock()
+	self.fwdCacheMu.Lock()
-	defer self.fwdCacheMu.RUnlock()
+	defer self.fwdCacheMu.Unlock()
+
+	digest := self.digest(msg)
 	entry, ok := self.fwdCache[digest]
 	if ok {
 		if entry.expiresAt.After(time.Now()) {
 			log.Trace(fmt.Sprintf("unexpired cache for digest %x", digest))
 			return true
-		} else if entry.expiresAt.IsZero() && bytes.Equal(addr, entry.receivedFrom) {
-			log.Trace(fmt.Sprintf("sendermatch %x for digest %x", common.ToHex(addr), digest))
-			return true
 		}
 	}
 	return false
 }
 
-// DPA storage handler for message cache
+// Digest of message
-func (self *Pss) storeMsg(msg *PssMsg) (pssDigest, error) {
+func (self *Pss) digest(msg *PssMsg) pssDigest {
-	buf := bytes.NewReader(msg.serialize())
+	hasher := self.hashPool.Get().(storage.SwarmHash)
-	key, _, err := self.dpa.Store(buf, int64(buf.Len()), false)
+	defer self.hashPool.Put(hasher)
-	if err != nil {
+	hasher.Reset()
-		log.Warn("Could not store in swarm", "err", err)
+	hasher.Write(msg.serialize())
-		return pssDigest{}, err
-	}
-	log.Trace("Stored msg in swarm", "key", key)
 	digest := pssDigest{}
+	key := hasher.Sum(nil)
 	copy(digest[:], key[:digestLength])
-	return digest, nil
+	return digest
 }
 
 func (self *Pss) isMsgExpired(msg *PssMsg) bool {

swarm/pss/pss_test.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/state"
-	"github.com/ethereum/go-ethereum/swarm/storage"
 	whisper "github.com/ethereum/go-ethereum/whisper/whisperv5"
 )
 
@@ -62,7 +61,7 @@ func init() {
 	flag.Parse()
 	rand.Seed(time.Now().Unix())
 
-	adapters.RegisterServices(newServices())
+	adapters.RegisterServices(newServices(false))
 	initTest()
 }
 
@@ -148,6 +147,7 @@ func TestCache(t *testing.T) {
 	pp := NewPssParams(privkey)
 	data := []byte("foo")
 	datatwo := []byte("bar")
+	datathree := []byte("baz")
 	wparams := &whisper.MessageParams{
 		TTL:      defaultWhisperTTL,
 		Src:      privkey,
@@ -170,38 +170,62 @@ func TestCache(t *testing.T) {
 		Payload: envtwo,
 		To:      to,
 	}
+	wparams.Payload = datathree
+	woutmsg, err = whisper.NewSentMessage(wparams)
+	envthree, err := woutmsg.Wrap(wparams)
+	msgthree := &PssMsg{
+		Payload: envthree,
+		To:      to,
+	}
 
-	digest, err := ps.storeMsg(msg)
+	digest := ps.digest(msg)
 	if err != nil {
 		t.Fatalf("could not store cache msgone: %v", err)
 	}
-	digesttwo, err := ps.storeMsg(msgtwo)
+	digesttwo := ps.digest(msgtwo)
 	if err != nil {
 		t.Fatalf("could not store cache msgtwo: %v", err)
 	}
+	digestthree := ps.digest(msgthree)
+	if err != nil {
+		t.Fatalf("could not store cache msgthree: %v", err)
+	}
 
 	if digest == digesttwo {
 		t.Fatalf("different msgs return same hash: %d", digesttwo)
 	}
 
 	// check the cache
-	err = ps.addFwdCache(digest)
+	err = ps.addFwdCache(msg)
 	if err != nil {
 		t.Fatalf("write to pss expire cache failed: %v", err)
 	}
 
-	if !ps.checkFwdCache(nil, digest) {
+	if !ps.checkFwdCache(msg) {
 		t.Fatalf("message %v should have EXPIRE record in cache but checkCache returned false", msg)
 	}
 
-	if ps.checkFwdCache(nil, digesttwo) {
+	if ps.checkFwdCache(msgtwo) {
 		t.Fatalf("message %v should NOT have EXPIRE record in cache but checkCache returned true", msgtwo)
 	}
 
-	time.Sleep(pp.CacheTTL)
+	time.Sleep(pp.CacheTTL + 1*time.Second)
-	if ps.checkFwdCache(nil, digest) {
+	err = ps.addFwdCache(msgthree)
+	if err != nil {
+		t.Fatalf("write to pss expire cache failed: %v", err)
+	}
+
+	if ps.checkFwdCache(msg) {
 		t.Fatalf("message %v should have expired from cache but checkCache returned true", msg)
 	}
+
+	if _, ok := ps.fwdCache[digestthree]; !ok {
+		t.Fatalf("unexpired message should be in the cache: %v", digestthree)
+	}
+
+	if _, ok := ps.fwdCache[digesttwo]; ok {
+		t.Fatalf("expired message should have been cleared from the cache: %v", digesttwo)
+	}
 }
 
 // matching of address hints; whether a message could be or is for the node
@@ -220,7 +244,7 @@ func TestAddressMatch(t *testing.T) {
 	}
 	privkey, err := w.GetPrivateKey(keys)
 	pssp := NewPssParams(privkey)
-	ps := NewPss(kad, nil, pssp)
+	ps := NewPss(kad, pssp)
 
 	pssmsg := &PssMsg{
 		To:      remoteaddr,
@@ -395,14 +419,96 @@ func TestMismatch(t *testing.T) {
 
 }
 
-// send symmetrically encrypted message between two directly connected peers
+func TestSendRaw(t *testing.T) {
-func TestSymSend(t *testing.T) {
+	t.Run("32", testSendRaw)
-	t.Run("32", testSymSend)
+	t.Run("8", testSendRaw)
-	t.Run("8", testSymSend)
+	t.Run("0", testSendRaw)
-	t.Run("0", testSymSend)
 }
 
-func testSymSend(t *testing.T) {
+func testSendRaw(t *testing.T) {
+
+	var addrsize int64
+	var err error
+
+	paramstring := strings.Split(t.Name(), "/")
+
+	addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0)
+	log.Info("raw send test", "addrsize", addrsize)
+
+	clients, err := setupNetwork(2, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	topic := "0x00000000"
+
+	var loaddrhex string
+	err = clients[0].Call(&loaddrhex, "pss_baseAddr")
+	if err != nil {
+		t.Fatalf("rpc get node 1 baseaddr fail: %v", err)
+	}
+	loaddrhex = loaddrhex[:2+(addrsize*2)]
+	var roaddrhex string
+	err = clients[1].Call(&roaddrhex, "pss_baseAddr")
+	if err != nil {
+		t.Fatalf("rpc get node 2 baseaddr fail: %v", err)
+	}
+	roaddrhex = roaddrhex[:2+(addrsize*2)]
+
+	time.Sleep(time.Millisecond * 500)
+
+	// at this point we've verified that symkeys are saved and match on each peer
+	// now try sending symmetrically encrypted message, both directions
+	lmsgC := make(chan APIMsg)
+	lctx, lcancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer lcancel()
+	lsub, err := clients[0].Subscribe(lctx, "pss", lmsgC, "receive", topic)
+	log.Trace("lsub", "id", lsub)
+	defer lsub.Unsubscribe()
+	rmsgC := make(chan APIMsg)
+	rctx, rcancel := context.WithTimeout(context.Background(), time.Second*10)
+	defer rcancel()
+	rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic)
+	log.Trace("rsub", "id", rsub)
+	defer rsub.Unsubscribe()
+
+	// send and verify delivery
+	lmsg := []byte("plugh")
+	err = clients[1].Call(nil, "pss_sendRaw", lmsg, loaddrhex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case recvmsg := <-lmsgC:
+		if !bytes.Equal(recvmsg.Msg, lmsg) {
+			t.Fatalf("node 1 received payload mismatch: expected %v, got %v", lmsg, recvmsg)
+		}
+	case cerr := <-lctx.Done():
+		t.Fatalf("test message (left) timed out: %v", cerr)
+	}
+	rmsg := []byte("xyzzy")
+	err = clients[0].Call(nil, "pss_sendRaw", rmsg, roaddrhex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case recvmsg := <-rmsgC:
+		if !bytes.Equal(recvmsg.Msg, rmsg) {
+			t.Fatalf("node 2 received payload mismatch: expected %x, got %v", rmsg, recvmsg.Msg)
+		}
+	case cerr := <-rctx.Done():
+		t.Fatalf("test message (right) timed out: %v", cerr)
+	}
+}
+
+// send symmetrically encrypted message between two directly connected peers
+func TestSendSym(t *testing.T) {
+	t.Run("32", testSendSym)
+	t.Run("8", testSendSym)
+	t.Run("0", testSendSym)
+}
+
+func testSendSym(t *testing.T) {
 
 	// address hint size
 	var addrsize int64
@@ -411,7 +517,7 @@ func testSymSend(t *testing.T) {
 	addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0)
 	log.Info("sym send test", "addrsize", addrsize)
 
-	clients, err := setupNetwork(2)
+	clients, err := setupNetwork(2, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -511,13 +617,13 @@ func testSymSend(t *testing.T) {
 }
 
 // send asymmetrically encrypted message between two directly connected peers
-func TestAsymSend(t *testing.T) {
+func TestSendAsym(t *testing.T) {
-	t.Run("32", testAsymSend)
+	t.Run("32", testSendAsym)
-	t.Run("8", testAsymSend)
+	t.Run("8", testSendAsym)
-	t.Run("0", testAsymSend)
+	t.Run("0", testSendAsym)
 }
 
-func testAsymSend(t *testing.T) {
+func testSendAsym(t *testing.T) {
 
 	// address hint size
 	var addrsize int64
@@ -526,7 +632,7 @@ func testAsymSend(t *testing.T) {
 	addrsize, _ = strconv.ParseInt(paramstring[1], 10, 0)
 	log.Info("asym send test", "addrsize", addrsize)
 
-	clients, err := setupNetwork(2)
+	clients, err := setupNetwork(2, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -686,11 +792,11 @@ func testNetwork(t *testing.T) {
 		}
 		a = adapters.NewExecAdapter(dirname)
 	} else if adapter == "sock" {
-		a = adapters.NewSocketAdapter(newServices())
+		a = adapters.NewSocketAdapter(newServices(false))
 	} else if adapter == "tcp" {
-		a = adapters.NewTCPAdapter(newServices())
+		a = adapters.NewTCPAdapter(newServices(false))
 	} else if adapter == "sim" {
-		a = adapters.NewSimAdapter(newServices())
+		a = adapters.NewSimAdapter(newServices(false))
 	}
 	net := simulations.NewNetwork(a, &simulations.NetworkConfig{
 		ID: "0",
@@ -840,6 +946,96 @@ outer:
 
 }
 
+// check that in a network of a -> b -> c -> a
+// a doesn't receive a sent message twice
+func TestDeduplication(t *testing.T) {
+	var err error
+
+	clients, err := setupNetwork(3, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var addrsize = 32
+	var loaddrhex string
+	err = clients[0].Call(&loaddrhex, "pss_baseAddr")
+	if err != nil {
+		t.Fatalf("rpc get node 1 baseaddr fail: %v", err)
+	}
+	loaddrhex = loaddrhex[:2+(addrsize*2)]
+	var roaddrhex string
+	err = clients[1].Call(&roaddrhex, "pss_baseAddr")
+	if err != nil {
+		t.Fatalf("rpc get node 2 baseaddr fail: %v", err)
+	}
+	roaddrhex = roaddrhex[:2+(addrsize*2)]
+	var xoaddrhex string
+	err = clients[2].Call(&xoaddrhex, "pss_baseAddr")
+	if err != nil {
+		t.Fatalf("rpc get node 3 baseaddr fail: %v", err)
+	}
+	xoaddrhex = xoaddrhex[:2+(addrsize*2)]
+
+	log.Info("peer", "l", loaddrhex, "r", roaddrhex, "x", xoaddrhex)
+
+	var topic string
+	err = clients[0].Call(&topic, "pss_stringToTopic", "foo:42")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	time.Sleep(time.Millisecond * 250)
+
+	// retrieve public key from pss instance
+	// set this public key reciprocally
+	var rpubkey string
+	err = clients[1].Call(&rpubkey, "pss_getPublicKey")
+	if err != nil {
+		t.Fatalf("rpc get receivenode pubkey fail: %v", err)
+	}
+
+	time.Sleep(time.Millisecond * 500) // replace with hive healthy code
+
+	rmsgC := make(chan APIMsg)
+	rctx, cancel := context.WithTimeout(context.Background(), time.Second*1)
+	defer cancel()
+	rsub, err := clients[1].Subscribe(rctx, "pss", rmsgC, "receive", topic)
+	log.Trace("rsub", "id", rsub)
+	defer rsub.Unsubscribe()
+
+	// store public key for recipient
+	// zero-length address means forward to all
+	// we have just two peers, they will be in proxbin, and will both receive
+	err = clients[0].Call(nil, "pss_setPeerPublicKey", rpubkey, topic, "0x")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// send and verify delivery
+	rmsg := []byte("xyzzy")
+	err = clients[0].Call(nil, "pss_sendAsym", rpubkey, topic, hexutil.Encode(rmsg))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var receivedok bool
+OUTER:
+	for {
+		select {
+		case <-rmsgC:
+			if receivedok {
+				t.Fatalf("duplicate message received")
+			}
+			receivedok = true
+		case <-rctx.Done():
+			break OUTER
+		}
+	}
+	if !receivedok {
+		t.Fatalf("message did not arrive")
+	}
+}
+
 // symmetric send performance with varying message sizes
 func BenchmarkSymkeySend(b *testing.B) {
 	b.Run(fmt.Sprintf("%d", 256), benchmarkSymKeySend)
@@ -996,7 +1192,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) {
 	}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		if !ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]) {
+		if err := ps.process(pssmsgs[len(pssmsgs)-(i%len(pssmsgs))-1]); err != nil {
 			b.Fatalf("pss processing failed: %v", err)
 		}
 	}
@@ -1078,20 +1274,22 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) {
 		Payload: env,
 	}
 	for i := 0; i < b.N; i++ {
-		if !ps.process(pssmsg) {
+		if err := ps.process(pssmsg); err != nil {
 			b.Fatalf("pss processing failed: %v", err)
 		}
 	}
 }
 
-// setup simulated network and connect nodes in circle
+// setup simulated network with bzz/discovery and pss services.
-func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
+// connects nodes in a circle
+// if allowRaw is set, omission of builtin pss encryption is enabled (see PssParams)
+func setupNetwork(numnodes int, allowRaw bool) (clients []*rpc.Client, err error) {
 	nodes := make([]*simulations.Node, numnodes)
 	clients = make([]*rpc.Client, numnodes)
 	if numnodes < 2 {
 		return nil, fmt.Errorf("Minimum two nodes in network")
 	}
-	adapter := adapters.NewSimAdapter(newServices())
+	adapter := adapters.NewSimAdapter(newServices(allowRaw))
 	net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{
 		ID:             "0",
 		DefaultService: "bzz",
@@ -1127,7 +1325,7 @@ func setupNetwork(numnodes int) (clients []*rpc.Client, err error) {
 	return clients, nil
 }
 
-func newServices() adapters.Services {
+func newServices(allowRaw bool) adapters.Services {
 	stateStore := state.NewInmemoryStore()
 	kademlias := make(map[discover.NodeID]*network.Kademlia)
 	kademlia := func(id discover.NodeID) *network.Kademlia {
@@ -1147,15 +1345,6 @@ func newServices() adapters.Services {
 	}
 	return adapters.Services{
 		pssProtocolName: func(ctx *adapters.ServiceContext) (node.Service, error) {
-			cachedir, err := ioutil.TempDir("", "pss-cache")
-			if err != nil {
-				return nil, fmt.Errorf("create pss cache tmpdir failed: %s", err)
-			}
-			dpa, err := storage.NewLocalDPA(cachedir, network.NewAddrFromNodeID(ctx.Config.ID).Over())
-			if err != nil {
-				return nil, fmt.Errorf("local dpa creation failed: %s", err)
-			}
-
 			// execadapter does not exec init()
 			initTest()
 
@@ -1165,8 +1354,9 @@ func newServices() adapters.Services {
 			privkey, err := w.GetPrivateKey(keys)
 			pssp := NewPssParams(privkey)
 			pssp.MsgTTL = time.Second * 30
+			pssp.AllowRaw = allowRaw
 			pskad := kademlia(ctx.Config.ID)
-			ps := NewPss(pskad, dpa, pssp)
+			ps := NewPss(pskad, pssp)
 
 			ping := &Ping{
 				OutC: make(chan bool),
@@ -1218,18 +1408,6 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss
 	copy(nid[:], crypto.FromECDSAPub(&privkey.PublicKey))
 	addr := network.NewAddrFromNodeID(nid)
 
-	// set up storage
-	cachedir, err := ioutil.TempDir("", "pss-cache")
-	if err != nil {
-		log.Error("create pss cache tmpdir failed", "error", err)
-		os.Exit(1)
-	}
-	dpa, err := storage.NewLocalDPA(cachedir, addr.Over())
-	if err != nil {
-		log.Error("local dpa creation failed", "error", err)
-		os.Exit(1)
-	}
-
 	// set up routing if kademlia is not passed to us
 	if overlay == nil {
 		kp := network.NewKadParams()
@@ -1242,7 +1420,8 @@ func newTestPss(privkey *ecdsa.PrivateKey, overlay network.Overlay, ppextra *Pss
 	if ppextra != nil {
 		pp.SymKeyCacheCapacity = ppextra.SymKeyCacheCapacity
 	}
-	ps := NewPss(overlay, dpa, pp)
+	ps := NewPss(overlay, pp)
+	ps.Start(nil)
 
 	return ps
 }

swarm/pss/types.go

@@ -20,6 +20,7 @@ const (
 var (
 	topicHashMutex = sync.Mutex{}
 	topicHashFunc  = storage.MakeHashFunc("SHA256")()
+	rawTopic       = Topic{}
 )
 
 type Topic whisper.TopicType
@@ -75,7 +76,13 @@ type PssMsg struct {
 
 // serializes the message for use in cache
 func (msg *PssMsg) serialize() []byte {
-	rlpdata, _ := rlp.EncodeToBytes(msg)
+	rlpdata, _ := rlp.EncodeToBytes(struct {
+		To      []byte
+		Payload *whisper.Envelope
+	}{
+		To:      msg.To,
+		Payload: msg.Payload,
+	})
 	return rlpdata
 }