swarm: merge branch 'master' into localstore-storage-integration

.travis.yml

@@ -14,7 +14,6 @@ matrix:
       - go run build/ci.go install
       - go run build/ci.go test -coverage $TEST_PACKAGES
 
-    # These are the latest Go versions.
     - os: linux
       dist: trusty
       sudo: required
@@ -26,8 +25,20 @@ matrix:
         - go run build/ci.go install
         - go run build/ci.go test -coverage $TEST_PACKAGES
 
+    # These are the latest Go versions.
+    - os: linux
+      dist: trusty
+      sudo: required
+      go: 1.12.x
+      script:
+        - sudo modprobe fuse
+        - sudo chmod 666 /dev/fuse
+        - sudo chown root:$USER /etc/fuse.conf
+        - go run build/ci.go install
+        - go run build/ci.go test -coverage $TEST_PACKAGES
+
     - os: osx
-      go: 1.11.x
+      go: 1.12.x
       script:
         - echo "Increase the maximum number of open file descriptors on macOS"
         - NOFILE=20480
@@ -44,7 +55,7 @@ matrix:
     # This builder only tests code linters on latest version of Go
     - os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - lint
       git:
@@ -56,7 +67,7 @@ matrix:
     - if: repo = ethereum/go-ethereum AND type = push
       os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - ubuntu-ppa
       git:
@@ -79,7 +90,7 @@ matrix:
       os: linux
       dist: trusty
       sudo: required
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-linux
       git:
@@ -114,7 +125,7 @@ matrix:
       dist: trusty
       services:
         - docker
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-linux-mips
       git:
@@ -159,7 +170,7 @@ matrix:
       git:
         submodules: false # avoid cloning ethereum/tests
       before_install:
-        - curl https://storage.googleapis.com/golang/go1.11.5.linux-amd64.tar.gz | tar -xz
+        - curl https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -xz
         - export PATH=`pwd`/go/bin:$PATH
         - export GOROOT=`pwd`/go
         - export GOPATH=$HOME/go
@@ -176,7 +187,7 @@ matrix:
     # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads
     - if: repo = ethereum/go-ethereum AND type = push
       os: osx
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-osx
         - azure-ios
@@ -206,7 +217,7 @@ matrix:
     - if: repo = ethereum/go-ethereum AND type = cron
       os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       env:
         - azure-purge
       git:
@@ -218,7 +229,7 @@ matrix:
       if: repo = ethersphere/go-ethereum
       os: linux
       dist: trusty
-      go: 1.11.x
+      go: 1.12.x
       git:
         submodules: false # avoid cloning ethereum/tests
       script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm...

Dockerfile

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.11-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

Dockerfile.alltools

@@ -1,5 +1,5 @@
 # Build Geth in a stock Go builder container
-FROM golang:1.11-alpine as builder
+FROM golang:1.12-alpine as builder
 
 RUN apk add --no-cache make gcc musl-dev linux-headers
 

appveyor.yml

@@ -23,8 +23,8 @@ environment:
 install:
   - git submodule update --init
   - rmdir C:\go /s /q
-  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.5.windows-%GETH_ARCH%.zip
-  - 7z x go1.11.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
+  - appveyor DownloadFile https://dl.google.com/go/go1.12.windows-%GETH_ARCH%.zip
+  - 7z x go1.12.windows-%GETH_ARCH%.zip -y -oC:\ > NUL
   - go version
   - gcc --version
 

build/ci.go

@@ -156,7 +156,7 @@ var (
 	// Note: yakkety is unsupported because it was officially deprecated on lanchpad.
 	// Note: zesty is unsupported because it was officially deprecated on lanchpad.
 	// Note: artful is unsupported because it was officially deprecated on lanchpad.
-	debDistros = []string{"trusty", "xenial", "bionic", "cosmic"}
+	debDistros = []string{"trusty", "xenial", "bionic", "cosmic", "disco"}
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))

build/deb/ethereum-swarm/deb.rules

@@ -4,6 +4,9 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
 	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 

build/deb/ethereum/deb.rules

@@ -4,6 +4,9 @@
 # Uncomment this to turn on verbose mode.
 #export DH_VERBOSE=1
 
+# Launchpad rejects Go's access to $HOME/.cache, use custom folder
+export GOCACHE=/tmp/go-build
+
 override_dh_auto_build:
 	build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}}
 

cmd/swarm/swarm-snapshot/create.go

@@ -59,13 +59,16 @@ func createSnapshot(filename string, nodes int, services []string) (err error) {
 	log.Debug("create snapshot", "filename", filename, "nodes", nodes, "services", services)
 
 	sim := simulation.New(map[string]simulation.ServiceFunc{
-		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
+		"bzz": func(ctx *adapters.ServiceContext, bucket *sync.Map) (node.Service, func(), error) {
 			addr := network.NewAddr(ctx.Config.Node())
 			kad := network.NewKademlia(addr.Over(), network.NewKadParams())
 			hp := network.NewHiveParams()
 			hp.KeepAliveInterval = time.Duration(200) * time.Millisecond
 			hp.Discovery = true // discovery must be enabled when creating a snapshot
 
+			// store the kademlia in the bucket, needed later in the WaitTillHealthy function
+			bucket.Store(simulation.BucketKeyKademlia, kad)
+
 			config := &network.BzzConfig{
 				OverlayAddr:  addr.Over(),
 				UnderlayAddr: addr.Under(),
@@ -76,17 +79,17 @@ func createSnapshot(filename string, nodes int, services []string) (err error) {
 	})
 	defer sim.Close()
 
-	_, err = sim.AddNodes(nodes)
+	ids, err := sim.AddNodes(nodes)
 	if err != nil {
 		return fmt.Errorf("add nodes: %v", err)
 	}
 
-	err = sim.Net.ConnectNodesRing(nil)
+	err = sim.Net.ConnectNodesRing(ids)
 	if err != nil {
 		return fmt.Errorf("connect nodes: %v", err)
 	}
 
-	ctx, cancelSimRun := context.WithTimeout(context.Background(), 2*time.Minute)
+	ctx, cancelSimRun := context.WithTimeout(context.Background(), 3*time.Minute)
 	defer cancelSimRun()
 	if _, err := sim.WaitTillHealthy(ctx); err != nil {
 		return fmt.Errorf("wait for healthy kademlia: %v", err)

cmd/swarm/swarm-snapshot/create_test.go

@@ -48,7 +48,7 @@ func TestSnapshotCreate(t *testing.T) {
 		},
 		{
 			name:  "more nodes",
-			nodes: defaultNodes + 5,
+			nodes: defaultNodes + 4,
 		},
 		{
 			name:     "services",
@@ -81,7 +81,7 @@ func TestSnapshotCreate(t *testing.T) {
 			}
 			testCmd := runSnapshot(t, append(args, file.Name())...)
 
-			testCmd.ExpectExit()
+			testCmd.WaitExit()
 			if code := testCmd.ExitStatus(); code != 0 {
 				t.Fatalf("command exit code %v, expected 0", code)
 			}

cmd/swarm/swarm-snapshot/main.go

@@ -27,7 +27,7 @@ import (
 var gitCommit string // Git SHA1 commit hash of the release (set via linker flags)
 
 // default value for "create" command --nodes flag
-const defaultNodes = 10
+const defaultNodes = 8
 
 func main() {
 	err := newApp().Run(os.Args)

common/fdlimit/fdlimit_darwin.go

@@ -0,0 +1,71 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package fdlimit
+
+import "syscall"
+
+// hardlimit is the number of file descriptors allowed at max by the kernel.
+const hardlimit = 10240
+
+// Raise tries to maximize the file descriptor allowance of this process
+// to the maximum hard-limit allowed by the OS.
+// Returns the size it was set to (may differ from the desired 'max')
+func Raise(max uint64) (uint64, error) {
+	// Get the current limit
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Try to update the limit to the max allowance
+	limit.Cur = limit.Max
+	if limit.Cur > max {
+		limit.Cur = max
+	}
+	if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// MacOS can silently apply further caps, so retrieve the actually set limit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return limit.Cur, nil
+}
+
+// Current retrieves the number of file descriptors allowed to be opened by this
+// process.
+func Current() (int, error) {
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	return int(limit.Cur), nil
+}
+
+// Maximum retrieves the maximum number of file descriptors this process is
+// allowed to request for itself.
+func Maximum() (int, error) {
+	// Retrieve the maximum allowed by dynamic OS limits
+	var limit syscall.Rlimit
+	if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
+		return 0, err
+	}
+	// Cap it to OPEN_MAX (10240) because macos is a special snowflake
+	if limit.Max > hardlimit {
+		limit.Max = hardlimit
+	}
+	return int(limit.Max), nil
+}

common/fdlimit/fdlimit_unix.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// +build linux darwin netbsd openbsd solaris
+// +build linux netbsd openbsd solaris
 
 package fdlimit
 

common/fdlimit/fdlimit_windows.go

@@ -18,6 +18,7 @@ package fdlimit
 
 import "fmt"
 
+// hardlimit is the number of file descriptors allowed at max by the kernel.
 const hardlimit = 16384
 
 // Raise tries to maximize the file descriptor allowance of this process

core/tx_pool_test.go

@@ -126,7 +126,7 @@ func validateEvents(events chan NewTxsEvent, count int) error {
 		case ev := <-events:
 			received = append(received, ev.Txs...)
 		case <-time.After(time.Second):
-			return fmt.Errorf("event #%d not fired", received)
+			return fmt.Errorf("event #%d not fired", len(received))
 		}
 	}
 	if len(received) > count {

core/types/log.go

@@ -115,8 +115,12 @@ func (l *LogForStorage) EncodeRLP(w io.Writer) error {
 //
 // Note some redundant fields(e.g. block number, tx hash etc) will be assembled later.
 func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
+	blob, err := s.Raw()
+	if err != nil {
+		return err
+	}
 	var dec rlpStorageLog
-	err := s.Decode(&dec)
+	err = rlp.DecodeBytes(blob, &dec)
 	if err == nil {
 		*l = LogForStorage{
 			Address: dec.Address,
@@ -126,7 +130,7 @@ func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
 	} else {
 		// Try to decode log with previous definition.
 		var dec LegacyRlpStorageLog
-		err = s.Decode(&dec)
+		err = rlp.DecodeBytes(blob, &dec)
 		if err == nil {
 			*l = LogForStorage{
 				Address: dec.Address,

core/types/receipt.go

@@ -184,10 +184,14 @@ func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
 // DecodeRLP implements rlp.Decoder, and loads both consensus and implementation
 // fields of a receipt from an RLP stream.
 func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
+	blob, err := s.Raw()
+	if err != nil {
+		return err
+	}
 	var dec receiptStorageRLP
-	if err := s.Decode(&dec); err != nil {
+	if err := rlp.DecodeBytes(blob, &dec); err != nil {
 		var sdec LegacyReceiptStorageRLP
-		if err := s.Decode(&sdec); err != nil {
+		if err := rlp.DecodeBytes(blob, &sdec); err != nil {
 			return err
 		}
 		dec.PostStateOrStatus = common.CopyBytes(sdec.PostStateOrStatus)

crypto/bn256/cloudflare/main_test.go

@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
 			t.Error(err)
 			continue
 		}
-		t.Logf("%d: %x\n", n, g2.Marshal())
+		t.Logf("%v: %x\n", n, g2.Marshal())
 	}
 }
 

crypto/bn256/cloudflare/mul_arm64.h

@@ -12,7 +12,7 @@
 	UMULH R1, R8, c4 \
 	ADCS ZR, c4 \
 	\
-	MUL R2, R5, R25 \
+	MUL R2, R5, R1 \
 	UMULH R2, R5, R26 \
 	MUL R2, R6, R0 \
 	ADDS R0, R26 \
@@ -24,13 +24,13 @@
 	ADCS R0, R29 \
 	UMULH R2, R8, c5 \
 	ADCS ZR, c5 \
-	ADDS R25, c1 \
+	ADDS R1, c1 \
 	ADCS R26, c2 \
 	ADCS R27, c3 \
 	ADCS R29, c4 \
 	ADCS  ZR, c5 \
 	\
-	MUL R3, R5, R25 \
+	MUL R3, R5, R1 \
 	UMULH R3, R5, R26 \
 	MUL R3, R6, R0 \
 	ADDS R0, R26 \
@@ -42,13 +42,13 @@
 	ADCS R0, R29 \
 	UMULH R3, R8, c6 \
 	ADCS ZR, c6 \
-	ADDS R25, c2 \
+	ADDS R1, c2 \
 	ADCS R26, c3 \
 	ADCS R27, c4 \
 	ADCS R29, c5 \
 	ADCS  ZR, c6 \
 	\
-	MUL R4, R5, R25 \
+	MUL R4, R5, R1 \
 	UMULH R4, R5, R26 \
 	MUL R4, R6, R0 \
 	ADDS R0, R26 \
@@ -60,7 +60,7 @@
 	ADCS R0, R29 \
 	UMULH R4, R8, c7 \
 	ADCS ZR, c7 \
-	ADDS R25, c3 \
+	ADDS R1, c3 \
 	ADCS R26, c4 \
 	ADCS R27, c5 \
 	ADCS R29, c6 \
@@ -69,15 +69,15 @@
 #define gfpReduce() \
 	\ // m = (T * N') mod R, store m in R1:R2:R3:R4
 	MOVD ·np+0(SB), R17 \
-	MOVD ·np+8(SB), R18 \
+	MOVD ·np+8(SB), R25 \
 	MOVD ·np+16(SB), R19 \
 	MOVD ·np+24(SB), R20 \
 	\
 	MUL R9, R17, R1 \
 	UMULH R9, R17, R2 \
-	MUL R9, R18, R0 \
+	MUL R9, R25, R0 \
 	ADDS R0, R2 \
-	UMULH R9, R18, R3 \
+	UMULH R9, R25, R3 \
 	MUL R9, R19, R0 \
 	ADCS R0, R3 \
 	UMULH R9, R19, R4 \
@@ -86,9 +86,9 @@
 	\
 	MUL R10, R17, R21 \
 	UMULH R10, R17, R22 \
-	MUL R10, R18, R0 \
+	MUL R10, R25, R0 \
 	ADDS R0, R22 \
-	UMULH R10, R18, R23 \
+	UMULH R10, R25, R23 \
 	MUL R10, R19, R0 \
 	ADCS R0, R23 \
 	ADDS R21, R2 \
@@ -97,7 +97,7 @@
 	\
 	MUL R11, R17, R21 \
 	UMULH R11, R17, R22 \
-	MUL R11, R18, R0 \
+	MUL R11, R25, R0 \
 	ADDS R0, R22 \
 	ADDS R21, R3 \
 	ADCS R22, R4 \
@@ -107,19 +107,19 @@
 	\
 	\ // m * N
 	loadModulus(R5,R6,R7,R8) \
-	mul(R17,R18,R19,R20,R21,R22,R23,R24) \
+	mul(R17,R25,R19,R20,R21,R22,R23,R24) \
 	\
 	\ // Add the 512-bit intermediate to m*N
-	MOVD  ZR, R25 \
+	MOVD  ZR, R0 \
 	ADDS  R9, R17 \
-	ADCS R10, R18 \
+	ADCS R10, R25 \
 	ADCS R11, R19 \
 	ADCS R12, R20 \
 	ADCS R13, R21 \
 	ADCS R14, R22 \
 	ADCS R15, R23 \
 	ADCS R16, R24 \
-	ADCS  ZR, R25 \
+	ADCS  ZR, R0 \
 	\
 	\ // Our output is R21:R22:R23:R24. Reduce mod p if necessary.
 	SUBS R5, R21, R10 \

crypto/bn256/google/main_test.go

@@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) {
 			t.Error(err)
 			continue
 		}
-		t.Logf("%d: %x\n", n, g2.Marshal())
+		t.Logf("%v: %x\n", n, g2.Marshal())
 	}
 }
 

internal/build/util.go

@@ -143,9 +143,9 @@ func CopyFile(dst, src string, mode os.FileMode) {
 // so that go commands executed by build use the same version of Go as the 'host' that runs
 // build code. e.g.
 //
-//     /usr/lib/go-1.11/bin/go run build/ci.go ...
+//     /usr/lib/go-1.12/bin/go run build/ci.go ...
 //
-// runs using go 1.11 and invokes go 1.11 tools from the same GOROOT. This is also important
+// runs using go 1.12 and invokes go 1.12 tools from the same GOROOT. This is also important
 // because runtime.Version checks on the host should match the tools that are run.
 func GoTool(tool string, args ...string) *exec.Cmd {
 	args = append([]string{tool}, args...)

node/defaults.go

@@ -80,13 +80,14 @@ func DefaultDataDir() string {
 }
 
 func windowsAppData() string {
-	if v := os.Getenv("LOCALAPPDATA"); v != "" {
-		return v // Vista+
+	v := os.Getenv("LOCALAPPDATA")
+	if v == "" {
+		// Windows XP and below don't have LocalAppData. Crash here because
+		// we don't support Windows XP and undefining the variable will cause
+		// other issues.
+		panic("environment variable LocalAppData is undefined")
 	}
-	if v := os.Getenv("APPDATA"); v != "" {
-		return filepath.Join(v, "Local")
-	}
-	return ""
+	return v
 }
 
 func isNonEmptyDir(dir string) bool {

p2p/discover/table_test.go

@@ -624,6 +624,8 @@ func (tn *preminedTestnet) findnode(toid enode.ID, toaddr *net.UDPAddr, target e
 func (*preminedTestnet) close()                                        {}
 func (*preminedTestnet) ping(toid enode.ID, toaddr *net.UDPAddr) error { return nil }
 
+var _ = (*preminedTestnet).mine // avoid linter warning about mine being dead code.
+
 // mine generates a testnet struct literal with nodes at
 // various distances to the given target.
 func (tn *preminedTestnet) mine(target encPubkey) {

p2p/discover/table_util_test.go

@@ -17,7 +17,6 @@
 package discover
 
 import (
-	"crypto/ecdsa"
 	"encoding/hex"
 	"fmt"
 	"math/rand"
@@ -167,11 +166,3 @@ func hexEncPubkey(h string) (ret encPubkey) {
 	copy(ret[:], b)
 	return ret
 }
-
-func hexPubkey(h string) *ecdsa.PublicKey {
-	k, err := decodePubkey(hexEncPubkey(h))
-	if err != nil {
-		panic(err)
-	}
-	return k
-}

p2p/enode/nodedb.go

@@ -57,7 +57,7 @@ const (
 const (
 	dbNodeExpiration = 24 * time.Hour // Time after which an unseen node should be dropped.
 	dbCleanupCycle   = time.Hour      // Time period for running the expiration task.
-	dbVersion        = 8
+	dbVersion        = 9
 )
 
 var zeroIP = make(net.IP, 16)
@@ -380,12 +380,12 @@ func (db *DB) UpdateFindFails(id ID, ip net.IP, fails int) error {
 
 // LocalSeq retrieves the local record sequence counter.
 func (db *DB) localSeq(id ID) uint64 {
-	return db.fetchUint64(nodeItemKey(id, zeroIP, dbLocalSeq))
+	return db.fetchUint64(localItemKey(id, dbLocalSeq))
 }
 
 // storeLocalSeq stores the local record sequence counter.
 func (db *DB) storeLocalSeq(id ID, n uint64) {
-	db.storeUint64(nodeItemKey(id, zeroIP, dbLocalSeq), n)
+	db.storeUint64(localItemKey(id, dbLocalSeq), n)
 }
 
 // QuerySeeds retrieves random nodes to be used as potential seed nodes

p2p/enode/nodedb_test.go

@@ -71,7 +71,7 @@ func TestDBNodeItemKey(t *testing.T) {
 	if id != keytestID {
 		t.Errorf("splitNodeItemKey returned wrong ID: %v", id)
 	}
-	if !bytes.Equal(ip, wantIP) {
+	if !ip.Equal(wantIP) {
 		t.Errorf("splitNodeItemKey returned wrong IP: %v", ip)
 	}
 	if field != wantField {

swarm/network/simulation/kademlia.go

@@ -58,7 +58,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*net
 			for k := range ill {
 				delete(ill, k)
 			}
-			log.Debug("kademlia health check", "addr count", len(addrs))
+			log.Debug("kademlia health check", "addr count", len(addrs), "kad len", len(kademlias))
 			for id, k := range kademlias {
 				//PeerPot for this node
 				addr := common.Bytes2Hex(k.BaseAddr())
@@ -70,7 +70,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*net
 				log.Debug("kademlia", "connectNN", h.ConnectNN, "knowNN", h.KnowNN)
 				log.Debug("kademlia", "health", h.ConnectNN && h.KnowNN, "addr", hex.EncodeToString(k.BaseAddr()), "node", id)
 				log.Debug("kademlia", "ill condition", !h.ConnectNN, "addr", hex.EncodeToString(k.BaseAddr()), "node", id)
-				if !h.ConnectNN {
+				if !h.Healthy() {
 					ill[id] = k
 				}
 			}
@@ -85,6 +85,7 @@ func (s *Simulation) WaitTillHealthy(ctx context.Context) (ill map[enode.ID]*net
 // in simulation bucket.
 func (s *Simulation) kademlias() (ks map[enode.ID]*network.Kademlia) {
 	items := s.UpNodesItems(BucketKeyKademlia)
+	log.Debug("kademlia len items", "len", len(items))
 	ks = make(map[enode.ID]*network.Kademlia, len(items))
 	for id, v := range items {
 		k, ok := v.(*network.Kademlia)

swarm/network/simulation/kademlia_test.go

@@ -22,16 +22,115 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/p2p/simulations/adapters"
 	"github.com/ethereum/go-ethereum/swarm/network"
 )
 
+/*
+	TestWaitTillHealthy tests that we indeed get a healthy network after we wait for it.
+	For this to be tested, a bit of a snake tail bite needs to happen:
+		* First we create a first simulation
+		* Run it as nodes connected in a ring
+		* Wait until the network is healthy
+		* Then we create a snapshot
+		* With this snapshot we create a new simulation
+		* This simulation is expected to have a healthy configuration, as it uses the snapshot
+		* Thus we just iterate all nodes and check that their kademlias are healthy
+		* If all kademlias are healthy, the test succeeded, otherwise it failed
+*/
 func TestWaitTillHealthy(t *testing.T) {
-	sim := New(map[string]ServiceFunc{
+
+	testNodesNum := 10
+
+	// create the first simulation
+	sim := New(createSimServiceMap(true))
+
+	// connect and...
+	nodeIDs, err := sim.AddNodesAndConnectRing(testNodesNum)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// array of all overlay addresses
+	var addrs [][]byte
+	// iterate once to be able to build the peer map
+	for _, node := range nodeIDs {
+		//get the kademlia overlay address from this ID
+		a := node.Bytes()
+		//append it to the array of all overlay addresses
+		addrs = append(addrs, a)
+	}
+	// build a PeerPot only once
+	pp := network.NewPeerPotMap(network.NewKadParams().NeighbourhoodSize, addrs)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	// ...wait until healthy
+	ill, err := sim.WaitTillHealthy(ctx)
+	if err != nil {
+		for id, kad := range ill {
+			t.Log("Node", id)
+			t.Log(kad.String())
+		}
+		t.Fatal(err)
+	}
+
+	// now create a snapshot of this network
+	snap, err := sim.Net.Snapshot()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// close the initial simulation
+	sim.Close()
+	// create a control simulation
+	controlSim := New(createSimServiceMap(false))
+	defer controlSim.Close()
+
+	// load the snapshot into this control simulation
+	err = controlSim.Net.Load(snap)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = controlSim.WaitTillHealthy(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, node := range nodeIDs {
+		// ...get its kademlia
+		item, ok := controlSim.NodeItem(node, BucketKeyKademlia)
+		if !ok {
+			t.Fatal("No kademlia bucket item")
+		}
+		kad := item.(*network.Kademlia)
+		// get its base address
+		kid := common.Bytes2Hex(kad.BaseAddr())
+
+		//get the health info
+		info := kad.GetHealthInfo(pp[kid])
+		log.Trace("Health info", "info", info)
+		// check that it is healthy
+		healthy := info.Healthy()
+		if !healthy {
+			t.Fatalf("Expected node %v of control simulation to be healthy, but it is not, unhealthy kademlias: %v", node, kad.String())
+		}
+	}
+}
+
+// createSimServiceMap returns the services map
+// this function will create the sim services with or without discovery enabled
+// based on the flag passed
+func createSimServiceMap(discovery bool) map[string]ServiceFunc {
+	return map[string]ServiceFunc{
 		"bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) {
 			addr := network.NewAddr(ctx.Config.Node())
 			hp := network.NewHiveParams()
+			hp.Discovery = discovery
 			config := &network.BzzConfig{
 				OverlayAddr:  addr.Over(),
 				UnderlayAddr: addr.Under(),
@@ -43,24 +142,5 @@ func TestWaitTillHealthy(t *testing.T) {
 			b.Store(BucketKeyKademlia, kad)
 			return network.NewBzz(config, kad, nil, nil, nil), nil, nil
 		},
-	})
-	defer sim.Close()
-
-	_, err := sim.AddNodesAndConnectRing(10)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
-	defer cancel()
-	ill, err := sim.WaitTillHealthy(ctx)
-	if err != nil {
-		for id, kad := range ill {
-			t.Log("Node", id)
-			t.Log(kad.String())
-		}
-		if err != nil {
-			t.Fatal(err)
-		}
 	}
 }

swarm/pss/testdata/addpsstodiscoverytestsnapshot.pl

@@ -1,28 +0,0 @@
-#!/usr/bin/perl
-
-use JSON;
-
-my $f;
-my $jsontext;
-my $nodelist;
-my $network;
-
-open($f, "<", $ARGV[0]) || die "cant open " . $ARGV[0];
-while (<$f>) {
-	$jsontext .= $_;
-}
-close($f);
-
-$network = decode_json($jsontext);
-$nodelist = $network->{'nodes'};
-
-for ($i = 0; $i < 0+@$nodelist; $i++) {
-	#my $protocollist = $$nodelist[$i]{'node'}{'info'}{'protocols'};
-	#$$protocollist{'pss'} = "pss";
-	my $svc = $$nodelist[$i]{'node'}{'config'}{'services'};
-	pop(@$svc);
-	push(@$svc, "pss");
-	push(@$svc, "bzz");
-}
-
-print encode_json($network);

swarm/pss/testdata/addpsstodiscoverytestsnapshot.sh

@@ -1,3 +0,0 @@
-#!/bin/bash
-
-sed -e 's/\(\"services\"\):\["discovery\"]/\1:["pss","bzz"]/'

swarm/pss/testdata/snapshot_2.json

@@ -1 +1 @@
-{"nodes":[{"node":{"config":{"id":"73d6ad4a75069dced660fa4cb98143ee5573df7cb15d9a295acf1655e9683384","private_key":"e567b7d9c554e5102cdc99b6523bace02dbb8951415c8816d82ba2d2e97fa23b","name":"node01","services":["pss","bzz"],"enable_msg_events":false,"port":0},"up":true}},{"node":{"config":{"id":"6e8da86abb894ab35044c8c455147225df96cab498da067a118f1fb9a417f9e3","private_key":"c7526db70acd02f36d3b201ef3e1d85e38c52bee6931453213dbc5edec4d0976","name":"node02","services":["pss","bzz"],"enable_msg_events":false,"port":0},"up":true}}],"conns":[{"one":"73d6ad4a75069dced660fa4cb98143ee5573df7cb15d9a295acf1655e9683384","other":"6e8da86abb894ab35044c8c455147225df96cab498da067a118f1fb9a417f9e3","up":true}]}
+{"nodes":[{"node":{"info":{"id":"fd37e45210ddb36f8b5ecef9c9d9d3f8201ba9523bfd6a70291333cf3b9207e1","name":"node_fd37e45210ddb36f8b5ecef9c9d9d3f8201ba9523bfd6a70291333cf3b9207e1","enode":"enode://6a7dd306ace8a106c3b86ac6b0cd6eeccde4a9af5090ca4a6d1054787f9757875b6b8d02f88fafe3fd7f20f0349a0060ace7c0be1d65470e736a3319431b5c14@127.0.0.1:0","enr":"0xf88fb84096f5dd9030fc74da3701ef7b06dfe4d44068118f8041a728876904d70fa8c88c20df197c343522c76d75d3a12507c567213c13b46a94ed4800f47474643f4ee70183636170cdc583627a7a08c6846869766508826964827634826970847f00000189736563703235366b31a1026a7dd306ace8a106c3b86ac6b0cd6eeccde4a9af5090ca4a6d1054787f975787","ip":"127.0.0.1","ports":{"discovery":0,"listener":0},"listenAddr":"","protocols":{"bzz":"/TfkUhDds2+LXs75ydnT+CAbqVI7/WpwKRMzzzuSB+E=","hive":"\n=========================================================================\nThu Feb 28 17:15:44 UTC 2019 KΛÐΞMLIΛ hive: queen's address: fd37e4\npopulation: 2 (2), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000  1 3594                         |  1 3594 (0)\n001  0                              |  0\n002  1 cb08                         |  1 cb08 (0)\n003  0                              |  0\n004  0                              |  0\n005  0                              |  0\n006  0                              |  0\n007  0                              |  0\n008  0                              |  0\n009  0                              |  0\n010  0                              |  0\n011  0                              |  0\n012  0                              |  0\n013  0                              |  0\n014  0                              |  0\n015  0                              |  0\n========================================================================="}},"config":{"id":"fd37e45210ddb36f8b5ecef9c9d9d3f8201ba9523bfd6a70291333cf3b9207e1","private_key":"6a02f87b1d13a15ce24bfa58456af8aaabdcf8cb58b9f86c19ab14c721c80f91","name":"node_fd37e45210ddb36f8b5ecef9c9d9d3f8201ba9523bfd6a70291333cf3b9207e1","services":["bzz","pss"],"enable_msg_events":true,"port":39341},"up":true}},{"node":{"info":{"id":"3594438a7480a5c3dc06440e28a16044c0c7d289219da02f9e41da1810827805","name":"node_3594438a7480a5c3dc06440e28a16044c0c7d289219da02f9e41da1810827805","enode":"enode://0972df8ba2504ddc155739320ab392e64d000c950e08ce42863449bc4757dd03397efbad98888f8d740b7cf55c357d1d6d74b5e8cb935f71b19e0e16ceef03f9@127.0.0.1:0","enr":"0xf88fb840d5f39b1ebcd5b616ef67b1c0239ed4501ef45c6a00e5bd8cd3f2ac88870d4c4b6ce4a4af820afab9a714d99a5fef53ec3c042fc3243f403819bcfb3504c2150d0183636170cdc583627a7a08c6846869766508826964827634826970847f00000189736563703235366b31a1030972df8ba2504ddc155739320ab392e64d000c950e08ce42863449bc4757dd03","ip":"127.0.0.1","ports":{"discovery":0,"listener":0},"listenAddr":"","protocols":{"bzz":"NZRDinSApcPcBkQOKKFgRMDH0okhnaAvnkHaGBCCeAU=","hive":"\n=========================================================================\nThu Feb 28 17:15:44 UTC 2019 KΛÐΞMLIΛ hive: queen's address: 359443\npopulation: 2 (2), NeighbourhoodSize: 2, MinBinSize: 2, MaxBinSize: 4\n============ DEPTH: 0 ==========================================\n000  2 fd37 cb08                    |  2 fd37 (0) cb08 (0)\n001  0                              |  0\n002  0                              |  0\n003  0                              |  0\n004  0                              |  0\n005  0                              |  0\n006  0                              |  0\n007  0                              |  0\n008  0                              |  0\n009  0                              |  0\n010  0                              |  0\n011  0                              |  0\n012  0                              |  0\n013  0                              |  0\n014  0                              |  0\n015  0                              |  0\n========================================================================="}},"config":{"id":"3594438a7480a5c3dc06440e28a16044c0c7d289219da02f9e41da1810827805","private_key":"26266470b7f4e9341cbbbb304b1aed116626cb1bea55d639cdbef1a069c18123","name":"node_3594438a7480a5c3dc06440e28a16044c0c7d289219da02f9e41da1810827805","services":["bzz","pss"],"enable_msg_events":true,"port":33055},"up":true}}],"conns":[{"one":"fd37e45210ddb36f8b5ecef9c9d9d3f8201ba9523bfd6a70291333cf3b9207e1","other":"3594438a7480a5c3dc06440e28a16044c0c7d289219da02f9e41da1810827805","up":true}]}

vendor/github.com/syndtr/goleveldb/leveldb/db.go

@@ -468,7 +468,7 @@ func recoverTable(s *session, o *opt.Options) error {
 	}
 
 	// Commit.
-	return s.commit(rec)
+	return s.commit(rec, false)
 }
 
 func (db *DB) recoverJournal() error {
@@ -538,7 +538,7 @@ func (db *DB) recoverJournal() error {
 
 				rec.setJournalNum(fd.Num)
 				rec.setSeqNum(db.seq)
-				if err := db.s.commit(rec); err != nil {
+				if err := db.s.commit(rec, false); err != nil {
 					fr.Close()
 					return err
 				}
@@ -617,7 +617,7 @@ func (db *DB) recoverJournal() error {
 	// Commit.
 	rec.setJournalNum(db.journalFd.Num)
 	rec.setSeqNum(db.seq)
-	if err := db.s.commit(rec); err != nil {
+	if err := db.s.commit(rec, false); err != nil {
 		// Close journal on error.
 		if db.journal != nil {
 			db.journal.Close()
@@ -872,6 +872,10 @@ func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
 // DB. And a nil Range.Limit is treated as a key after all keys in
 // the DB.
 //
+// WARNING: Any slice returned by interator (e.g. slice returned by calling
+// Iterator.Key() or Iterator.Key() methods), its content should not be modified
+// unless noted otherwise.
+//
 // The iterator must be released after use, by calling Release method.
 //
 // Also read Iterator documentation of the leveldb/iterator package.
@@ -953,15 +957,27 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 		value = "Compactions\n" +
 			" Level |   Tables   |    Size(MB)   |    Time(sec)  |    Read(MB)   |   Write(MB)\n" +
 			"-------+------------+---------------+---------------+---------------+---------------\n"
+		var totalTables int
+		var totalSize, totalRead, totalWrite int64
+		var totalDuration time.Duration
 		for level, tables := range v.levels {
 			duration, read, write := db.compStats.getStat(level)
 			if len(tables) == 0 && duration == 0 {
 				continue
 			}
+			totalTables += len(tables)
+			totalSize += tables.size()
+			totalRead += read
+			totalWrite += write
+			totalDuration += duration
 			value += fmt.Sprintf(" %3d   | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
 				level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(),
 				float64(read)/1048576.0, float64(write)/1048576.0)
 		}
+		value += "-------+------------+---------------+---------------+---------------+---------------\n"
+		value += fmt.Sprintf(" Total | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
+			totalTables, float64(totalSize)/1048576.0, totalDuration.Seconds(),
+			float64(totalRead)/1048576.0, float64(totalWrite)/1048576.0)
 	case p == "iostats":
 		value = fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f",
 			float64(db.s.stor.reads())/1048576.0,
@@ -1013,10 +1029,10 @@ type DBStats struct {
 	BlockCacheSize    int
 	OpenedTablesCount int
 
-	LevelSizes        []int64
+	LevelSizes        Sizes
 	LevelTablesCounts []int
-	LevelRead         []int64
-	LevelWrite        []int64
+	LevelRead         Sizes
+	LevelWrite        Sizes
 	LevelDurations    []time.Duration
 }
 

vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -260,7 +260,7 @@ func (db *DB) compactionCommit(name string, rec *sessionRecord) {
 	db.compCommitLk.Lock()
 	defer db.compCommitLk.Unlock() // Defer is necessary.
 	db.compactionTransactFunc(name+"@commit", func(cnt *compactionTransactCounter) error {
-		return db.s.commit(rec)
+		return db.s.commit(rec, true)
 	}, nil)
 }
 

vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -142,6 +142,10 @@ func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error)
 // DB. And a nil Range.Limit is treated as a key after all keys in
 // the DB.
 //
+// WARNING: Any slice returned by interator (e.g. slice returned by calling
+// Iterator.Key() or Iterator.Value() methods), its content should not be
+// modified unless noted otherwise.
+//
 // The iterator must be released after use, by calling Release method.
 // Releasing the snapshot doesn't mean releasing the iterator too, the
 // iterator would be still valid until released.

vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go

@@ -69,6 +69,10 @@ func (tr *Transaction) Has(key []byte, ro *opt.ReadOptions) (bool, error) {
 // DB. And a nil Range.Limit is treated as a key after all keys in
 // the DB.
 //
+// WARNING: Any slice returned by interator (e.g. slice returned by calling
+// Iterator.Key() or Iterator.Key() methods), its content should not be modified
+// unless noted otherwise.
+//
 // The iterator must be released after use, by calling Release method.
 //
 // Also read Iterator documentation of the leveldb/iterator package.
@@ -205,7 +209,7 @@ func (tr *Transaction) Commit() error {
 		tr.stats.startTimer()
 		var cerr error
 		for retry := 0; retry < 3; retry++ {
-			cerr = tr.db.s.commit(&tr.rec)
+			cerr = tr.db.s.commit(&tr.rec, false)
 			if cerr != nil {
 				tr.db.logf("transaction@commit error R·%d %q", retry, cerr)
 				select {

vendor/github.com/syndtr/goleveldb/leveldb/session.go

@@ -180,19 +180,19 @@ func (s *session) recover() (err error) {
 	}
 
 	s.manifestFd = fd
-	s.setVersion(staging.finish())
+	s.setVersion(staging.finish(false))
 	s.setNextFileNum(rec.nextFileNum)
 	s.recordCommited(rec)
 	return nil
 }
 
 // Commit session; need external synchronization.
-func (s *session) commit(r *sessionRecord) (err error) {
+func (s *session) commit(r *sessionRecord, trivial bool) (err error) {
 	v := s.version()
 	defer v.release()
 
 	// spawn new version based on current version
-	nv := v.spawn(r)
+	nv := v.spawn(r, trivial)
 
 	if s.manifest == nil {
 		// manifest journal writer not yet created, create one

vendor/github.com/syndtr/goleveldb/leveldb/table.go

@@ -150,6 +150,14 @@ func (tf tFiles) searchMax(icmp *iComparer, ikey internalKey) int {
 	})
 }
 
+// Searches smallest index of tables whose its file number
+// is smaller than the given number.
+func (tf tFiles) searchNumLess(num int64) int {
+	return sort.Search(len(tf), func(i int) bool {
+		return tf[i].fd.Num < num
+	})
+}
+
 // Returns true if given key range overlaps with one or more
 // tables key range. If unsorted is true then binary search will not be used.
 func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool {

vendor/github.com/syndtr/goleveldb/leveldb/version.go

@@ -273,10 +273,10 @@ func (v *version) newStaging() *versionStaging {
 }
 
 // Spawn a new version based on this version.
-func (v *version) spawn(r *sessionRecord) *version {
+func (v *version) spawn(r *sessionRecord, trivial bool) *version {
 	staging := v.newStaging()
 	staging.commit(r)
-	return staging.finish()
+	return staging.finish(trivial)
 }
 
 func (v *version) fillRecord(r *sessionRecord) {
@@ -446,7 +446,7 @@ func (p *versionStaging) commit(r *sessionRecord) {
 	}
 }
 
-func (p *versionStaging) finish() *version {
+func (p *versionStaging) finish(trivial bool) *version {
 	// Build new version.
 	nv := newVersion(p.base.s)
 	numLevel := len(p.levels)
@@ -463,6 +463,12 @@ func (p *versionStaging) finish() *version {
 		if level < len(p.levels) {
 			scratch := p.levels[level]
 
+			// Short circuit if there is no change at all.
+			if len(scratch.added) == 0 && len(scratch.deleted) == 0 {
+				nv.levels[level] = baseTabels
+				continue
+			}
+
 			var nt tFiles
 			// Prealloc list if possible.
 			if n := len(baseTabels) + len(scratch.added) - len(scratch.deleted); n > 0 {
@@ -480,6 +486,35 @@ func (p *versionStaging) finish() *version {
 				nt = append(nt, t)
 			}
 
+			// For normal table compaction, one compaction will only involve two levels
+			// of files. And the new files generated after merging the source level and
+			// source+1 level related files can be inserted as a whole into source+1 level
+			// without any overlap with the other source+1 files.
+			//
+			// When the amount of data maintained by leveldb is large, the number of files
+			// per level will be very large. While qsort is very inefficient for sorting
+			// already ordered arrays. Therefore, for the normal table compaction, we use
+			// binary search here to find the insert index to insert a batch of new added
+			// files directly instead of using qsort.
+			if trivial && len(scratch.added) > 0 {
+				added := make(tFiles, 0, len(scratch.added))
+				for _, r := range scratch.added {
+					added = append(added, tableFileFromRecord(r))
+				}
+				if level == 0 {
+					added.sortByNum()
+					index := nt.searchNumLess(added[len(added)-1].fd.Num)
+					nt = append(nt[:index], append(added, nt[index:]...)...)
+				} else {
+					added.sortByKey(p.base.s.icmp)
+					_, amax := added.getRange(p.base.s.icmp)
+					index := nt.searchMin(p.base.s.icmp, amax)
+					nt = append(nt[:index], append(added, nt[index:]...)...)
+				}
+				nv.levels[level] = nt
+				continue
+			}
+
 			// New tables.
 			for _, r := range scratch.added {
 				nt = append(nt, tableFileFromRecord(r))

vendor/vendor.json

@@ -455,10 +455,10 @@
 			"revisionTime": "2017-07-05T02:17:15Z"
 		},
 		{
-			"checksumSHA1": "LV0VMVON7xY1ttV+s2ph83ntmDQ=",
+			"checksumSHA1": "4DuP8qJfeXFfdbcl4wr7l1VppcY=",
 			"path": "github.com/syndtr/goleveldb/leveldb",
-			"revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae",
-			"revisionTime": "2018-11-28T10:09:59Z"
+			"revision": "4217c9f31f5816db02addc94e56061da77f288d8",
+			"revisionTime": "2019-02-26T15:37:22Z"
 		},
 		{
 			"checksumSHA1": "mPNraL2edpk/2FYq26rSXfMHbJg=",