From 1993227311f2e67398a5796f3d5a0dbe48c4409f Mon Sep 17 00:00:00 2001 From: Nguyen Kien Trung Date: Fri, 22 Feb 2019 06:07:04 -0500 Subject: [PATCH 01/21] .travis.yml: make build independent of repo name (#17607) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a079d9eb8f..015807dc1f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -171,7 +171,7 @@ matrix: - export ANDROID_NDK=$HOME/android-ndk-r17b - mkdir -p $GOPATH/src/github.com/ethereum - - ln -s `pwd` $GOPATH/src/github.com/ethereum + - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads From d9adcd3a27cb14042bdb230f073487192683390c Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Fri, 22 Feb 2019 14:20:21 +0100 Subject: [PATCH 02/21] travis.yml: add race detector job for Swarm (#19148) --- .travis.yml | 21 +++++++++++++----- build/travis_keepalive.sh | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) create mode 100755 build/travis_keepalive.sh diff --git a/.travis.yml b/.travis.yml index 015807dc1f..5d61391138 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,7 @@ matrix: - go run build/ci.go lint # This builder does the Ubuntu PPA upload - - if: type = push + - if: repo = ethereum/go-ethereum AND type = push os: linux dist: trusty go: 1.11.x @@ -75,7 +75,7 @@ matrix: - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - - if: type = push + - if: repo = ethereum/go-ethereum AND type = push os: linux dist: trusty sudo: required @@ -109,7 +109,7 @@ matrix: - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # This builder does the Linux Azure MIPS xgo uploads - - if: type = push + - if: repo = ethereum/go-ethereum AND type = push os: linux dist: trusty services: @@ -137,7 +137,7 @@ matrix: - go run build/ci.go archive -arch mips64le -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds # This builder does the Android Maven and Azure uploads - - if: type = push + - if: repo = ethereum/go-ethereum AND type = push os: linux dist: trusty addons: @@ -175,7 +175,7 @@ matrix: - go run build/ci.go aar -signer ANDROID_SIGNING_KEY -deploy https://oss.sonatype.org -upload gethstore/builds # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads - - if: type = push + - if: repo = ethereum/go-ethereum AND type = push os: osx go: 1.11.x env: @@ -204,7 +204,7 @@ matrix: - go run build/ci.go xcode -signer IOS_SIGNING_KEY -deploy trunk -upload gethstore/builds # This builder does the Azure archive purges to avoid accumulating junk - - if: type = cron + - if: repo = ethereum/go-ethereum AND type = cron os: linux dist: trusty go: 1.11.x @@ -214,3 +214,12 @@ matrix: submodules: false # avoid cloning ethereum/tests script: - go run build/ci.go purge -store gethstore/builds -days 14 + + - name: Race Detector for Swarm + if: repo = ethersphere/go-ethereum + os: linux + dist: trusty + go: 1.11.x + git: + submodules: false # avoid cloning ethereum/tests + script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm... diff --git a/build/travis_keepalive.sh b/build/travis_keepalive.sh new file mode 100755 index 0000000000..77cc623eaf --- /dev/null +++ b/build/travis_keepalive.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# travis_keepalive runs the given command and preserves its return value, +# while it forks a child process what periodically produces a log line, +# so that Travis won't abort the build after 10 minutes. + +# Why? +# `t.Log()` in Go holds the buffer until the test does not pass or fail, +# and `-race` can increase the execution time by 2-20x. + +set -euo pipefail + +readonly KEEPALIVE_INTERVAL=300 # seconds => 5m + +main() { + keepalive + $@ +} + +# Keepalive produces a log line in each KEEPALIVE_INTERVAL. +keepalive() { + local child_pid + # Note: We fork here! + repeat "keepalive" & + child_pid=$! + ensureChildOnEXIT "${child_pid}" +} + +repeat() { + local this="$1" + while true; do + echo "${this}" + sleep "${KEEPALIVE_INTERVAL}" + done +} + +# Ensures that the child gets killed on normal program exit. +ensureChildOnEXIT() { + # Note: SIGINT and SIGTERM are forwarded to the child process by Bash + # automatically, so we don't have to deal with signals. + + local child_pid="$1" + trap "kill ${child_pid}" EXIT +} + +main "$@" From 02c28046a04ebf649af5d1b2a702d0da1c8a2a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Fri, 22 Feb 2019 23:19:09 +0100 Subject: [PATCH 03/21] swarm: Fix localstore test deadlock with race detector (#19153) * swarm/storage/localstore: close localstore in two tests * swarm/storage/localstore: fix a possible deadlock in tests * swarm/storage/localstore: re-enable pull subs tests for travis race * swarm/storage/localstore: stop sending to errChan on context done in tests * swarm/storage/localstore: better want check in readPullSubscriptionBin * swarm/storage/localstore: protect chunk put with addr lock in tests * swamr/storage/localstore: wait for gc and writeGCSize workers on Close * swarm/storage/localstore: more correct testDB_collectGarbageWorker * swarm/storage/localstore: set DB Close timeout to 5s --- swarm/storage/localstore/gc.go | 6 +- swarm/storage/localstore/gc_test.go | 10 +-- swarm/storage/localstore/localstore.go | 29 +++++++- swarm/storage/localstore/localstore_test.go | 1 + .../localstore/subscription_pull_test.go | 71 +++++++------------ .../localstore/subscription_push_test.go | 22 ++++-- 6 files changed, 73 insertions(+), 66 deletions(-) diff --git a/swarm/storage/localstore/gc.go b/swarm/storage/localstore/gc.go index 7718d1e589..ebaba2d8f3 100644 --- a/swarm/storage/localstore/gc.go +++ b/swarm/storage/localstore/gc.go @@ -117,6 +117,8 @@ var ( // run. GC run iterates on gcIndex and removes older items // form retrieval and other indexes. func (db *DB) collectGarbageWorker() { + defer close(db.collectGarbageWorkerDone) + for { select { case <-db.collectGarbageTrigger: @@ -132,7 +134,7 @@ func (db *DB) collectGarbageWorker() { db.triggerGarbageCollection() } - if testHookCollectGarbage != nil { + if collectedCount > 0 && testHookCollectGarbage != nil { testHookCollectGarbage(collectedCount) } case <-db.close: @@ -243,6 +245,8 @@ func (db *DB) triggerGarbageCollection() { // writeGCSizeDelay duration to avoid very frequent // database operations. func (db *DB) writeGCSizeWorker() { + defer close(db.writeGCSizeWorkerDone) + for { select { case <-db.writeGCSizeTrigger: diff --git a/swarm/storage/localstore/gc_test.go b/swarm/storage/localstore/gc_test.go index 322b846654..60309d7fa9 100644 --- a/swarm/storage/localstore/gc_test.go +++ b/swarm/storage/localstore/gc_test.go @@ -118,15 +118,6 @@ func testDB_collectGarbageWorker(t *testing.T) { t.Fatal(err) } }) - - // cleanup: drain the last testHookCollectGarbageChan - // element before calling deferred functions not to block - // collectGarbageWorker loop, preventing the race in - // setting testHookCollectGarbage function - select { - case <-testHookCollectGarbageChan: - default: - } } // TestDB_collectGarbageWorker_withRequests is a helper test function @@ -290,6 +281,7 @@ func TestDB_gcSize(t *testing.T) { if err != nil { t.Fatal(err) } + defer db.Close() t.Run("gc index size", newIndexGCSizeTest(db)) diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index 7a9fb54f55..f92a9c1f2a 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -107,6 +107,12 @@ type DB struct { // this channel is closed when close function is called // to terminate other goroutines close chan struct{} + + // protect Close method from exiting before + // garbage collection and gc size write workers + // are done + collectGarbageWorkerDone chan struct{} + writeGCSizeWorkerDone chan struct{} } // Options struct holds optional parameters for configuring DB. @@ -138,9 +144,11 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { // need to be buffered with the size of 1 // to signal another event if it // is triggered during already running function - collectGarbageTrigger: make(chan struct{}, 1), - writeGCSizeTrigger: make(chan struct{}, 1), - close: make(chan struct{}), + collectGarbageTrigger: make(chan struct{}, 1), + writeGCSizeTrigger: make(chan struct{}, 1), + close: make(chan struct{}), + collectGarbageWorkerDone: make(chan struct{}), + writeGCSizeWorkerDone: make(chan struct{}), } if db.capacity <= 0 { db.capacity = defaultCapacity @@ -361,6 +369,21 @@ func New(path string, baseKey []byte, o *Options) (db *DB, err error) { func (db *DB) Close() (err error) { close(db.close) db.updateGCWG.Wait() + + // wait for gc worker and gc size write workers to + // return before closing the shed + timeout := time.After(5 * time.Second) + select { + case <-db.collectGarbageWorkerDone: + case <-timeout: + log.Error("localstore: collect garbage worker did not return after db close") + } + select { + case <-db.writeGCSizeWorkerDone: + case <-timeout: + log.Error("localstore: write gc size worker did not return after db close") + } + if err := db.writeGCSize(db.getGCSize()); err != nil { log.Error("localstore: write gc size", "err", err) } diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index c7309d3cd8..6b48d54e98 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -163,6 +163,7 @@ func BenchmarkNew(b *testing.B) { if err != nil { b.Fatal(err) } + defer db.Close() uploader := db.NewPutter(ModePutUpload) syncer := db.NewSetter(ModeSetSync) for i := 0; i < count; i++ { diff --git a/swarm/storage/localstore/subscription_pull_test.go b/swarm/storage/localstore/subscription_pull_test.go index 16fe8b0ddc..9800329eaa 100644 --- a/swarm/storage/localstore/subscription_pull_test.go +++ b/swarm/storage/localstore/subscription_pull_test.go @@ -20,13 +20,11 @@ import ( "bytes" "context" "fmt" - "os" "sync" "testing" "time" "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/testutil" ) // TestDB_SubscribePull uploads some chunks before and after @@ -34,12 +32,6 @@ import ( // all addresses are received in the right order // for expected proximity order bins. func TestDB_SubscribePull(t *testing.T) { - - if testutil.RaceEnabled && os.Getenv("TRAVIS") == "true" { - t.Skip("does not complete with -race on Travis") - // Note: related ticket TODO - } - db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() @@ -87,12 +79,6 @@ func TestDB_SubscribePull(t *testing.T) { // validates if all addresses are received in the right order // for expected proximity order bins. func TestDB_SubscribePull_multiple(t *testing.T) { - - if testutil.RaceEnabled && os.Getenv("TRAVIS") == "true" { - t.Skip("does not complete with -race on Travis") - // Note: related ticket TODO - } - db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() @@ -146,12 +132,6 @@ func TestDB_SubscribePull_multiple(t *testing.T) { // and validates if all expected addresses are received in the // right order for expected proximity order bins. func TestDB_SubscribePull_since(t *testing.T) { - - if testutil.RaceEnabled && os.Getenv("TRAVIS") == "true" { - t.Skip("does not complete with -race on Travis") - // Note: related ticket TODO - } - db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() @@ -171,6 +151,9 @@ func TestDB_SubscribePull_since(t *testing.T) { })() uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + addrsMu.Lock() + defer addrsMu.Unlock() + last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -182,7 +165,6 @@ func TestDB_SubscribePull_since(t *testing.T) { bin := db.po(chunk.Address()) - addrsMu.Lock() if _, ok := addrs[bin]; !ok { addrs[bin] = make([]storage.Address, 0) } @@ -190,7 +172,6 @@ func TestDB_SubscribePull_since(t *testing.T) { addrs[bin] = append(addrs[bin], chunk.Address()) wantedChunksCount++ } - addrsMu.Unlock() lastTimestampMu.RLock() storeTimestamp := lastTimestamp @@ -242,12 +223,6 @@ func TestDB_SubscribePull_since(t *testing.T) { // and validates if all expected addresses are received in the // right order for expected proximity order bins. func TestDB_SubscribePull_until(t *testing.T) { - - if testutil.RaceEnabled && os.Getenv("TRAVIS") == "true" { - t.Skip("does not complete with -race on Travis") - // Note: related ticket TODO - } - db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() @@ -267,6 +242,9 @@ func TestDB_SubscribePull_until(t *testing.T) { })() uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + addrsMu.Lock() + defer addrsMu.Unlock() + last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -278,7 +256,6 @@ func TestDB_SubscribePull_until(t *testing.T) { bin := db.po(chunk.Address()) - addrsMu.Lock() if _, ok := addrs[bin]; !ok { addrs[bin] = make([]storage.Address, 0) } @@ -286,7 +263,6 @@ func TestDB_SubscribePull_until(t *testing.T) { addrs[bin] = append(addrs[bin], chunk.Address()) wantedChunksCount++ } - addrsMu.Unlock() lastTimestampMu.RLock() storeTimestamp := lastTimestamp @@ -337,12 +313,6 @@ func TestDB_SubscribePull_until(t *testing.T) { // and until arguments, and validates if all expected addresses // are received in the right order for expected proximity order bins. func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { - - if testutil.RaceEnabled && os.Getenv("TRAVIS") == "true" { - t.Skip("does not complete with -race on Travis") - // Note: related ticket TODO - } - db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() @@ -362,6 +332,9 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { })() uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + addrsMu.Lock() + defer addrsMu.Unlock() + last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -373,7 +346,6 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { bin := db.po(chunk.Address()) - addrsMu.Lock() if _, ok := addrs[bin]; !ok { addrs[bin] = make([]storage.Address, 0) } @@ -381,7 +353,6 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { addrs[bin] = append(addrs[bin], chunk.Address()) wantedChunksCount++ } - addrsMu.Unlock() lastTimestampMu.RLock() storeTimestamp := lastTimestamp @@ -442,6 +413,9 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { // uploadRandomChunksBin uploads random chunks to database and adds them to // the map of addresses ber bin. func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) { + addrsMu.Lock() + defer addrsMu.Unlock() + for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -450,13 +424,11 @@ func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uin t.Fatal(err) } - addrsMu.Lock() bin := db.po(chunk.Address()) if _, ok := addrs[bin]; !ok { addrs[bin] = make([]storage.Address, 0) } addrs[bin] = append(addrs[bin], chunk.Address()) - addrsMu.Unlock() *wantedChunksCount++ } @@ -473,19 +445,24 @@ func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDesc if !ok { return } + var err error addrsMu.Lock() if i+1 > len(addrs[bin]) { - errChan <- fmt.Errorf("got more chunk addresses %v, then expected %v, for bin %v", i+1, len(addrs[bin]), bin) + err = fmt.Errorf("got more chunk addresses %v, then expected %v, for bin %v", i+1, len(addrs[bin]), bin) + } else { + want := addrs[bin][i] + if !bytes.Equal(got.Address, want) { + err = fmt.Errorf("got chunk address %v in bin %v %s, want %s", i, bin, got.Address.Hex(), want) + } } - want := addrs[bin][i] addrsMu.Unlock() - var err error - if !bytes.Equal(got.Address, want) { - err = fmt.Errorf("got chunk address %v in bin %v %s, want %s", i, bin, got.Address.Hex(), want) - } i++ // send one and only one error per received address - errChan <- err + select { + case errChan <- err: + case <-ctx.Done(): + return + } case <-ctx.Done(): return } diff --git a/swarm/storage/localstore/subscription_push_test.go b/swarm/storage/localstore/subscription_push_test.go index 73e7c25f72..0c8d7d0b9a 100644 --- a/swarm/storage/localstore/subscription_push_test.go +++ b/swarm/storage/localstore/subscription_push_test.go @@ -40,6 +40,9 @@ func TestDB_SubscribePush(t *testing.T) { var chunksMu sync.Mutex uploadRandomChunks := func(count int) { + chunksMu.Lock() + defer chunksMu.Unlock() + for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -48,9 +51,7 @@ func TestDB_SubscribePush(t *testing.T) { t.Fatal(err) } - chunksMu.Lock() chunks = append(chunks, chunk) - chunksMu.Unlock() } } @@ -90,7 +91,11 @@ func TestDB_SubscribePush(t *testing.T) { } i++ // send one and only one error per received address - errChan <- err + select { + case errChan <- err: + case <-ctx.Done(): + return + } case <-ctx.Done(): return } @@ -123,6 +128,9 @@ func TestDB_SubscribePush_multiple(t *testing.T) { var addrsMu sync.Mutex uploadRandomChunks := func(count int) { + addrsMu.Lock() + defer addrsMu.Unlock() + for i := 0; i < count; i++ { chunk := generateRandomChunk() @@ -131,9 +139,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) { t.Fatal(err) } - addrsMu.Lock() addrs = append(addrs, chunk.Address()) - addrsMu.Unlock() } } @@ -175,7 +181,11 @@ func TestDB_SubscribePush_multiple(t *testing.T) { } i++ // send one and only one error per received address - errChan <- err + select { + case errChan <- err: + case <-ctx.Done(): + return + } case <-ctx.Done(): return } From 64d10c08726af33048e8eeb8df257628a3944870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Sat, 23 Feb 2019 10:47:33 +0100 Subject: [PATCH 04/21] swarm: mock store listings (#19157) * swarm/storage/mock: implement listings methods for mem and rpc stores * swarm/storage/mock/rpc: add comments and newTestStore helper function * swarm/storage/mock/mem: add missing comments * swarm/storage/mock: add comments to new types and constants * swarm/storage/mock/db: implement listings for mock/db global store * swarm/storage/mock/test: add comments for MockStoreListings * swarm/storage/mock/explorer: initial implementation * cmd/swarm/global-store: add chunk explorer * cmd/swarm/global-store: add chunk explorer tests * swarm/storage/mock/explorer: add tests * swarm/storage/mock/explorer: add swagger api definition * swarm/storage/mock/explorer: not-zero test values for invalid addr and key * swarm/storage/mock/explorer: test wildcard cors origin * swarm/storage/mock/db: renames based on Fabio's suggestions * swarm/storage/mock/explorer: add more comments to testHandler function * cmd/swarm/global-store: terminate subprocess with Kill in tests --- cmd/swarm/global-store/explorer.go | 66 +++ cmd/swarm/global-store/explorer_test.go | 254 ++++++++++ cmd/swarm/global-store/global_store.go | 24 +- cmd/swarm/global-store/global_store_test.go | 64 ++- cmd/swarm/global-store/main.go | 44 +- swarm/storage/mock/db/db.go | 295 ++++++++++-- swarm/storage/mock/db/db_test.go | 70 +-- swarm/storage/mock/explorer/explorer.go | 257 ++++++++++ swarm/storage/mock/explorer/explorer_test.go | 471 +++++++++++++++++++ swarm/storage/mock/explorer/headers_test.go | 163 +++++++ swarm/storage/mock/explorer/swagger.yaml | 176 +++++++ swarm/storage/mock/mem/mem.go | 270 +++++++++-- swarm/storage/mock/mem/mem_test.go | 6 + swarm/storage/mock/mock.go | 31 ++ swarm/storage/mock/rpc/rpc.go | 24 + swarm/storage/mock/rpc/rpc_test.go | 29 +- swarm/storage/mock/test/test.go | 118 +++++ 17 files changed, 2215 insertions(+), 147 deletions(-) create mode 100644 cmd/swarm/global-store/explorer.go create mode 100644 cmd/swarm/global-store/explorer_test.go create mode 100644 swarm/storage/mock/explorer/explorer.go create mode 100644 swarm/storage/mock/explorer/explorer_test.go create mode 100644 swarm/storage/mock/explorer/headers_test.go create mode 100644 swarm/storage/mock/explorer/swagger.yaml diff --git a/cmd/swarm/global-store/explorer.go b/cmd/swarm/global-store/explorer.go new file mode 100644 index 0000000000..634ff1ebb6 --- /dev/null +++ b/cmd/swarm/global-store/explorer.go @@ -0,0 +1,66 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "fmt" + "net" + "net/http" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/explorer" + cli "gopkg.in/urfave/cli.v1" +) + +// serveChunkExplorer starts an http server in background with chunk explorer handler +// using the provided global store. Server is started if the returned shutdown function +// is not nil. +func serveChunkExplorer(ctx *cli.Context, globalStore mock.GlobalStorer) (shutdown func(), err error) { + if !ctx.IsSet("explorer-address") { + return nil, nil + } + + corsOrigins := ctx.StringSlice("explorer-cors-origin") + server := &http.Server{ + Handler: explorer.NewHandler(globalStore, corsOrigins), + IdleTimeout: 30 * time.Minute, + ReadTimeout: 2 * time.Minute, + WriteTimeout: 2 * time.Minute, + } + listener, err := net.Listen("tcp", ctx.String("explorer-address")) + if err != nil { + return nil, fmt.Errorf("explorer: %v", err) + } + log.Info("chunk explorer http", "address", listener.Addr().String(), "origins", corsOrigins) + + go func() { + if err := server.Serve(listener); err != nil { + log.Error("chunk explorer", "err", err) + } + }() + + return func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(ctx); err != nil { + log.Error("chunk explorer: shutdown", "err", err) + } + }, nil +} diff --git a/cmd/swarm/global-store/explorer_test.go b/cmd/swarm/global-store/explorer_test.go new file mode 100644 index 0000000000..2e4928c8fc --- /dev/null +++ b/cmd/swarm/global-store/explorer_test.go @@ -0,0 +1,254 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "sort" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage/mock/explorer" + mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" +) + +// TestExplorer validates basic chunk explorer functionality by storing +// a small set of chunk and making http requests on exposed endpoint. +// Full chunk explorer validation is done in mock/explorer package. +func TestExplorer(t *testing.T) { + addr := findFreeTCPAddress(t) + explorerAddr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, "ws", "--addr", addr, "--explorer-address", explorerAddr) + defer testCmd.Kill() + + client := websocketClient(t, addr) + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + nodeKeys := map[string][]string{ + "a1": {"b1", "b2", "b3"}, + "a2": {"b3", "b4", "b5"}, + } + + keyNodes := make(map[string][]string) + + for addr, keys := range nodeKeys { + for _, key := range keys { + keyNodes[key] = append(keyNodes[key], addr) + } + } + + invalidAddr := "c1" + invalidKey := "d1" + + for addr, keys := range nodeKeys { + for _, key := range keys { + err := store.Put(common.HexToAddress(addr), common.Hex2Bytes(key), []byte("data")) + if err != nil { + t.Fatal(err) + } + } + } + + endpoint := "http://" + explorerAddr + + t.Run("has key", func(t *testing.T) { + for addr, keys := range nodeKeys { + for _, key := range keys { + testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+key, http.StatusOK) + testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+key, http.StatusNotFound) + } + testStatusResponse(t, endpoint+"/api/has-key/"+addr+"/"+invalidKey, http.StatusNotFound) + } + testStatusResponse(t, endpoint+"/api/has-key/"+invalidAddr+"/"+invalidKey, http.StatusNotFound) + }) + + t.Run("keys", func(t *testing.T) { + var keys []string + for key := range keyNodes { + keys = append(keys, key) + } + sort.Strings(keys) + testKeysResponse(t, endpoint+"/api/keys", explorer.KeysResponse{ + Keys: keys, + }) + }) + + t.Run("nodes", func(t *testing.T) { + var nodes []string + for addr := range nodeKeys { + nodes = append(nodes, common.HexToAddress(addr).Hex()) + } + sort.Strings(nodes) + testNodesResponse(t, endpoint+"/api/nodes", explorer.NodesResponse{ + Nodes: nodes, + }) + }) + + t.Run("node keys", func(t *testing.T) { + for addr, keys := range nodeKeys { + testKeysResponse(t, endpoint+"/api/keys?node="+addr, explorer.KeysResponse{ + Keys: keys, + }) + } + testKeysResponse(t, endpoint+"/api/keys?node="+invalidAddr, explorer.KeysResponse{}) + }) + + t.Run("key nodes", func(t *testing.T) { + for key, addrs := range keyNodes { + var nodes []string + for _, addr := range addrs { + nodes = append(nodes, common.HexToAddress(addr).Hex()) + } + sort.Strings(nodes) + testNodesResponse(t, endpoint+"/api/nodes?key="+key, explorer.NodesResponse{ + Nodes: nodes, + }) + } + testNodesResponse(t, endpoint+"/api/nodes?key="+invalidKey, explorer.NodesResponse{}) + }) +} + +// TestExplorer_CORSOrigin validates if chunk explorer returns +// correct CORS origin header in GET and OPTIONS requests. +func TestExplorer_CORSOrigin(t *testing.T) { + origin := "http://localhost/" + addr := findFreeTCPAddress(t) + explorerAddr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, "ws", + "--addr", addr, + "--explorer-address", explorerAddr, + "--explorer-cors-origin", origin, + ) + defer testCmd.Kill() + + // wait until the server is started + waitHTTPEndpoint(t, explorerAddr) + + url := "http://" + explorerAddr + "/api/keys" + + t.Run("get", func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", origin) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + header := resp.Header.Get("Access-Control-Allow-Origin") + if header != origin { + t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin) + } + }) + + t.Run("preflight", func(t *testing.T) { + req, err := http.NewRequest(http.MethodOptions, url, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", origin) + req.Header.Set("Access-Control-Request-Method", "GET") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + header := resp.Header.Get("Access-Control-Allow-Origin") + if header != origin { + t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, origin) + } + }) +} + +// testStatusResponse makes an http request to provided url +// and validates if response is explorer.StatusResponse for +// the expected status code. +func testStatusResponse(t *testing.T, url string, code int) { + t.Helper() + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != code { + t.Errorf("got status code %v, want %v", resp.StatusCode, code) + } + var r explorer.StatusResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if r.Code != code { + t.Errorf("got response code %v, want %v", r.Code, code) + } + if r.Message != http.StatusText(code) { + t.Errorf("got response message %q, want %q", r.Message, http.StatusText(code)) + } +} + +// testKeysResponse makes an http request to provided url +// and validates if response machhes expected explorer.KeysResponse. +func testKeysResponse(t *testing.T, url string, want explorer.KeysResponse) { + t.Helper() + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK) + } + var r explorer.KeysResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if fmt.Sprint(r.Keys) != fmt.Sprint(want.Keys) { + t.Errorf("got keys %v, want %v", r.Keys, want.Keys) + } + if r.Next != want.Next { + t.Errorf("got next %s, want %s", r.Next, want.Next) + } +} + +// testNodeResponse makes an http request to provided url +// and validates if response machhes expected explorer.NodeResponse. +func testNodesResponse(t *testing.T, url string, want explorer.NodesResponse) { + t.Helper() + + resp, err := http.Get(url) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK) + } + var r explorer.NodesResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if fmt.Sprint(r.Nodes) != fmt.Sprint(want.Nodes) { + t.Errorf("got nodes %v, want %v", r.Nodes, want.Nodes) + } + if r.Next != want.Next { + t.Errorf("got next %s, want %s", r.Next, want.Next) + } +} diff --git a/cmd/swarm/global-store/global_store.go b/cmd/swarm/global-store/global_store.go index a55756e1c4..f93b464dbc 100644 --- a/cmd/swarm/global-store/global_store.go +++ b/cmd/swarm/global-store/global_store.go @@ -17,6 +17,7 @@ package main import ( + "io" "net" "net/http" "os" @@ -66,7 +67,7 @@ func startWS(ctx *cli.Context) (err error) { return http.Serve(listener, server.WebsocketHandler(origins)) } -// newServer creates a global store and returns its RPC server. +// newServer creates a global store and starts a chunk explorer server if configured. // Returned cleanup function should be called only if err is nil. func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) { log.PrintOrigins(true) @@ -81,7 +82,9 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) return nil, nil, err } cleanup = func() { - dbStore.Close() + if err := dbStore.Close(); err != nil { + log.Error("global store: close", "err", err) + } } globalStore = dbStore log.Info("database global store", "dir", dir) @@ -96,5 +99,22 @@ func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) return nil, nil, err } + shutdown, err := serveChunkExplorer(ctx, globalStore) + if err != nil { + cleanup() + return nil, nil, err + } + if shutdown != nil { + cleanup = func() { + shutdown() + + if c, ok := globalStore.(io.Closer); ok { + if err := c.Close(); err != nil { + log.Error("global store: close", "err", err) + } + } + } + } + return server, cleanup, nil } diff --git a/cmd/swarm/global-store/global_store_test.go b/cmd/swarm/global-store/global_store_test.go index 63f1c5389e..c437c9d453 100644 --- a/cmd/swarm/global-store/global_store_test.go +++ b/cmd/swarm/global-store/global_store_test.go @@ -69,16 +69,7 @@ func testHTTP(t *testing.T, put bool, args ...string) { // wait until global store process is started as // rpc.DialHTTP is actually not connecting - for i := 0; i < 1000; i++ { - _, err = http.DefaultClient.Get("http://" + addr) - if err == nil { - break - } - time.Sleep(10 * time.Millisecond) - } - if err != nil { - t.Fatal(err) - } + waitHTTPEndpoint(t, addr) store := mockRPC.NewGlobalStore(client) defer store.Close() @@ -137,19 +128,7 @@ func testWebsocket(t *testing.T, put bool, args ...string) { testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...) defer testCmd.Kill() - var client *rpc.Client - var err error - // wait until global store process is started - for i := 0; i < 1000; i++ { - client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") - if err == nil { - break - } - time.Sleep(10 * time.Millisecond) - } - if err != nil { - t.Fatal(err) - } + client := websocketClient(t, addr) store := mockRPC.NewGlobalStore(client) defer store.Close() @@ -160,7 +139,7 @@ func testWebsocket(t *testing.T, put bool, args ...string) { wantValue := "value" if put { - err = node.Put([]byte(wantKey), []byte(wantValue)) + err := node.Put([]byte(wantKey), []byte(wantValue)) if err != nil { t.Fatal(err) } @@ -189,3 +168,40 @@ func findFreeTCPAddress(t *testing.T) (addr string) { return listener.Addr().String() } + +// websocketClient waits until global store process is started +// and returns rpc client. +func websocketClient(t *testing.T, addr string) (client *rpc.Client) { + t.Helper() + + var err error + for i := 0; i < 1000; i++ { + client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + return client +} + +// waitHTTPEndpoint retries http requests to a provided +// address until the connection is established. +func waitHTTPEndpoint(t *testing.T, addr string) { + t.Helper() + + var err error + for i := 0; i < 1000; i++ { + _, err = http.Get("http://" + addr) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/swarm/global-store/main.go b/cmd/swarm/global-store/main.go index 51df0099ac..52fafc8f6a 100644 --- a/cmd/swarm/global-store/main.go +++ b/cmd/swarm/global-store/main.go @@ -19,12 +19,14 @@ package main import ( "os" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/log" cli "gopkg.in/urfave/cli.v1" ) -var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +var ( + version = "0.1" + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +) func main() { err := newApp().Run(os.Args) @@ -37,16 +39,30 @@ func main() { // newApp construct a new instance of Swarm Global Store. // Method Run is called on it in the main function and in tests. func newApp() (app *cli.App) { - app = utils.NewApp(gitCommit, "Swarm Global Store") - + app = cli.NewApp() app.Name = "global-store" + app.Version = version + if len(gitCommit) >= 8 { + app.Version += "-" + gitCommit[:8] + } + app.Usage = "Swarm Global Store" // app flags (for all commands) app.Flags = []cli.Flag{ cli.IntFlag{ Name: "verbosity", Value: 3, - Usage: "verbosity level", + Usage: "Verbosity level.", + }, + cli.StringFlag{ + Name: "explorer-address", + Value: "", + Usage: "Chunk explorer HTTP listener address.", + }, + cli.StringSliceFlag{ + Name: "explorer-cors-origin", + Value: nil, + Usage: "Chunk explorer CORS origin (can be specified multiple times).", }, } @@ -54,7 +70,7 @@ func newApp() (app *cli.App) { { Name: "http", Aliases: []string{"h"}, - Usage: "start swarm global store with http server", + Usage: "Start swarm global store with HTTP server.", Action: startHTTP, // Flags only for "start" command. // Allow app flags to be specified after the @@ -63,19 +79,19 @@ func newApp() (app *cli.App) { cli.StringFlag{ Name: "dir", Value: "", - Usage: "data directory", + Usage: "Data directory.", }, cli.StringFlag{ Name: "addr", Value: "0.0.0.0:3033", - Usage: "address to listen for http connection", + Usage: "Address to listen for HTTP connections.", }, ), }, { Name: "websocket", Aliases: []string{"ws"}, - Usage: "start swarm global store with websocket server", + Usage: "Start swarm global store with WebSocket server.", Action: startWS, // Flags only for "start" command. // Allow app flags to be specified after the @@ -84,17 +100,17 @@ func newApp() (app *cli.App) { cli.StringFlag{ Name: "dir", Value: "", - Usage: "data directory", + Usage: "Data directory.", }, cli.StringFlag{ Name: "addr", Value: "0.0.0.0:3033", - Usage: "address to listen for websocket connection", + Usage: "Address to listen for WebSocket connections.", }, cli.StringSliceFlag{ - Name: "origins", - Value: &cli.StringSlice{"*"}, - Usage: "websocket origins", + Name: "origin", + Value: nil, + Usage: "WebSocket CORS origin (can be specified multiple times).", }, ), }, diff --git a/swarm/storage/mock/db/db.go b/swarm/storage/mock/db/db.go index 73ae199e8b..313a61b43e 100644 --- a/swarm/storage/mock/db/db.go +++ b/swarm/storage/mock/db/db.go @@ -21,8 +21,12 @@ import ( "archive/tar" "bytes" "encoding/json" + "errors" + "fmt" "io" "io/ioutil" + "sync" + "time" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/util" @@ -37,6 +41,10 @@ import ( // release resources used by the database. type GlobalStore struct { db *leveldb.DB + // protects nodes and keys indexes + // in Put and Delete methods + nodesLocks sync.Map + keysLocks sync.Map } // NewGlobalStore creates a new instance of GlobalStore. @@ -64,14 +72,14 @@ func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore { // Get returns chunk data if the chunk with key exists for node // on address addr. func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) { - has, err := s.db.Has(nodeDBKey(addr, key), nil) + has, err := s.db.Has(indexForHashesPerNode(addr, key), nil) if err != nil { return nil, mock.ErrNotFound } if !has { return nil, mock.ErrNotFound } - data, err = s.db.Get(dataDBKey(key), nil) + data, err = s.db.Get(indexDataKey(key), nil) if err == leveldb.ErrNotFound { err = mock.ErrNotFound } @@ -80,28 +88,165 @@ func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err err // Put saves the chunk data for node with address addr. func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { + unlock, err := s.lock(addr, key) + if err != nil { + return err + } + defer unlock() + batch := new(leveldb.Batch) - batch.Put(nodeDBKey(addr, key), nil) - batch.Put(dataDBKey(key), data) + batch.Put(indexForHashesPerNode(addr, key), nil) + batch.Put(indexForNodesWithHash(key, addr), nil) + batch.Put(indexForNodes(addr), nil) + batch.Put(indexForHashes(key), nil) + batch.Put(indexDataKey(key), data) return s.db.Write(batch, nil) } // Delete removes the chunk reference to node with address addr. func (s *GlobalStore) Delete(addr common.Address, key []byte) error { + unlock, err := s.lock(addr, key) + if err != nil { + return err + } + defer unlock() + batch := new(leveldb.Batch) - batch.Delete(nodeDBKey(addr, key)) + batch.Delete(indexForHashesPerNode(addr, key)) + batch.Delete(indexForNodesWithHash(key, addr)) + + // check if this node contains any keys, and if not + // remove it from the + x := indexForHashesPerNodePrefix(addr) + if k, _ := s.db.Get(x, nil); !bytes.HasPrefix(k, x) { + batch.Delete(indexForNodes(addr)) + } + + x = indexForNodesWithHashPrefix(key) + if k, _ := s.db.Get(x, nil); !bytes.HasPrefix(k, x) { + batch.Delete(indexForHashes(key)) + } return s.db.Write(batch, nil) } // HasKey returns whether a node with addr contains the key. func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { - has, err := s.db.Has(nodeDBKey(addr, key), nil) + has, err := s.db.Has(indexForHashesPerNode(addr, key), nil) if err != nil { has = false } return has } +// Keys returns a paginated list of keys on all nodes. +func (s *GlobalStore) Keys(startKey []byte, limit int) (keys mock.Keys, err error) { + return s.keys(nil, startKey, limit) +} + +// Nodes returns a paginated list of all known nodes. +func (s *GlobalStore) Nodes(startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + return s.nodes(nil, startAddr, limit) +} + +// NodeKeys returns a paginated list of keys on a node with provided address. +func (s *GlobalStore) NodeKeys(addr common.Address, startKey []byte, limit int) (keys mock.Keys, err error) { + return s.keys(&addr, startKey, limit) +} + +// KeyNodes returns a paginated list of nodes that contain a particular key. +func (s *GlobalStore) KeyNodes(key []byte, startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + return s.nodes(key, startAddr, limit) +} + +// keys returns a paginated list of keys. If addr is not nil, only keys on that +// node will be returned. +func (s *GlobalStore) keys(addr *common.Address, startKey []byte, limit int) (keys mock.Keys, err error) { + iter := s.db.NewIterator(nil, nil) + defer iter.Release() + + if limit <= 0 { + limit = mock.DefaultLimit + } + + prefix := []byte{indexForHashesPrefix} + if addr != nil { + prefix = indexForHashesPerNodePrefix(*addr) + } + if startKey != nil { + if addr != nil { + startKey = indexForHashesPerNode(*addr, startKey) + } else { + startKey = indexForHashes(startKey) + } + } else { + startKey = prefix + } + + ok := iter.Seek(startKey) + if !ok { + return keys, iter.Error() + } + for ; ok; ok = iter.Next() { + k := iter.Key() + if !bytes.HasPrefix(k, prefix) { + break + } + key := append([]byte(nil), bytes.TrimPrefix(k, prefix)...) + + if len(keys.Keys) >= limit { + keys.Next = key + break + } + + keys.Keys = append(keys.Keys, key) + } + return keys, iter.Error() +} + +// nodes returns a paginated list of node addresses. If key is not nil, +// only nodes that contain that key will be returned. +func (s *GlobalStore) nodes(key []byte, startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + iter := s.db.NewIterator(nil, nil) + defer iter.Release() + + if limit <= 0 { + limit = mock.DefaultLimit + } + + prefix := []byte{indexForNodesPrefix} + if key != nil { + prefix = indexForNodesWithHashPrefix(key) + } + startKey := prefix + if startAddr != nil { + if key != nil { + startKey = indexForNodesWithHash(key, *startAddr) + } else { + startKey = indexForNodes(*startAddr) + } + } + + ok := iter.Seek(startKey) + if !ok { + return nodes, iter.Error() + } + for ; ok; ok = iter.Next() { + k := iter.Key() + if !bytes.HasPrefix(k, prefix) { + break + } + addr := common.BytesToAddress(append([]byte(nil), bytes.TrimPrefix(k, prefix)...)) + + if len(nodes.Addrs) >= limit { + nodes.Next = &addr + break + } + + nodes.Addrs = append(nodes.Addrs, addr) + } + return nodes, iter.Error() +} + // Import reads tar archive from a reader that contains exported chunk data. // It returns the number of chunks imported and an error. func (s *GlobalStore) Import(r io.Reader) (n int, err error) { @@ -126,12 +271,18 @@ func (s *GlobalStore) Import(r io.Reader) (n int, err error) { return n, err } + key := common.Hex2Bytes(hdr.Name) + batch := new(leveldb.Batch) for _, addr := range c.Addrs { - batch.Put(nodeDBKeyHex(addr, hdr.Name), nil) + batch.Put(indexForHashesPerNode(addr, key), nil) + batch.Put(indexForNodesWithHash(key, addr), nil) + batch.Put(indexForNodes(addr), nil) } - batch.Put(dataDBKey(common.Hex2Bytes(hdr.Name)), c.Data) + batch.Put(indexForHashes(key), nil) + batch.Put(indexDataKey(key), c.Data) + if err = s.db.Write(batch, nil); err != nil { return n, err } @@ -150,18 +301,23 @@ func (s *GlobalStore) Export(w io.Writer) (n int, err error) { buf := bytes.NewBuffer(make([]byte, 0, 1024)) encoder := json.NewEncoder(buf) - iter := s.db.NewIterator(util.BytesPrefix(nodeKeyPrefix), nil) + snap, err := s.db.GetSnapshot() + if err != nil { + return 0, err + } + + iter := snap.NewIterator(util.BytesPrefix([]byte{indexForHashesByNodePrefix}), nil) defer iter.Release() var currentKey string var addrs []common.Address - saveChunk := func(hexKey string) error { - key := common.Hex2Bytes(hexKey) + saveChunk := func() error { + hexKey := currentKey - data, err := s.db.Get(dataDBKey(key), nil) + data, err := snap.Get(indexDataKey(common.Hex2Bytes(hexKey)), nil) if err != nil { - return err + return fmt.Errorf("get data %s: %v", hexKey, err) } buf.Reset() @@ -189,8 +345,8 @@ func (s *GlobalStore) Export(w io.Writer) (n int, err error) { } for iter.Next() { - k := bytes.TrimPrefix(iter.Key(), nodeKeyPrefix) - i := bytes.Index(k, []byte("-")) + k := bytes.TrimPrefix(iter.Key(), []byte{indexForHashesByNodePrefix}) + i := bytes.Index(k, []byte{keyTermByte}) if i < 0 { continue } @@ -201,7 +357,7 @@ func (s *GlobalStore) Export(w io.Writer) (n int, err error) { } if hexKey != currentKey { - if err = saveChunk(currentKey); err != nil { + if err = saveChunk(); err != nil { return n, err } @@ -209,35 +365,112 @@ func (s *GlobalStore) Export(w io.Writer) (n int, err error) { } currentKey = hexKey - addrs = append(addrs, common.BytesToAddress(k[i:])) + addrs = append(addrs, common.BytesToAddress(k[i+1:])) } if len(addrs) > 0 { - if err = saveChunk(currentKey); err != nil { + if err = saveChunk(); err != nil { return n, err } } - return n, err + return n, iter.Error() } var ( - nodeKeyPrefix = []byte("node-") - dataKeyPrefix = []byte("data-") + // maximal time for lock to wait until it returns error + lockTimeout = 3 * time.Second + // duration between two lock checks. + lockCheckDelay = 30 * time.Microsecond + // error returned by lock method when lock timeout is reached + errLockTimeout = errors.New("lock timeout") ) -// nodeDBKey constructs a database key for key/node mappings. -func nodeDBKey(addr common.Address, key []byte) []byte { - return nodeDBKeyHex(addr, common.Bytes2Hex(key)) +// lock protects parallel writes in Put and Delete methods for both +// node with provided address and for data with provided key. +func (s *GlobalStore) lock(addr common.Address, key []byte) (unlock func(), err error) { + start := time.Now() + nodeLockKey := addr.Hex() + for { + _, loaded := s.nodesLocks.LoadOrStore(nodeLockKey, struct{}{}) + if !loaded { + break + } + time.Sleep(lockCheckDelay) + if time.Since(start) > lockTimeout { + return nil, errLockTimeout + } + } + start = time.Now() + keyLockKey := common.Bytes2Hex(key) + for { + _, loaded := s.keysLocks.LoadOrStore(keyLockKey, struct{}{}) + if !loaded { + break + } + time.Sleep(lockCheckDelay) + if time.Since(start) > lockTimeout { + return nil, errLockTimeout + } + } + return func() { + s.nodesLocks.Delete(nodeLockKey) + s.keysLocks.Delete(keyLockKey) + }, nil } -// nodeDBKeyHex constructs a database key for key/node mappings -// using the hexadecimal string representation of the key. -func nodeDBKeyHex(addr common.Address, hexKey string) []byte { - return append(append(nodeKeyPrefix, []byte(hexKey+"-")...), addr[:]...) +const ( + // prefixes for different indexes + indexDataPrefix = 0 + indexForNodesWithHashesPrefix = 1 + indexForHashesByNodePrefix = 2 + indexForNodesPrefix = 3 + indexForHashesPrefix = 4 + + // keyTermByte splits keys and node addresses + // in database keys + keyTermByte = 0xff +) + +// indexForHashesPerNode constructs a database key to store keys used in +// NodeKeys method. +func indexForHashesPerNode(addr common.Address, key []byte) []byte { + return append(indexForHashesPerNodePrefix(addr), key...) } -// dataDBkey constructs a database key for key/data storage. -func dataDBKey(key []byte) []byte { - return append(dataKeyPrefix, key...) +// indexForHashesPerNodePrefix returns a prefix containing a node address used in +// NodeKeys method. Node address is hex encoded to be able to use keyTermByte +// for splitting node address and key. +func indexForHashesPerNodePrefix(addr common.Address) []byte { + return append([]byte{indexForNodesWithHashesPrefix}, append([]byte(addr.Hex()), keyTermByte)...) +} + +// indexForNodesWithHash constructs a database key to store keys used in +// KeyNodes method. +func indexForNodesWithHash(key []byte, addr common.Address) []byte { + return append(indexForNodesWithHashPrefix(key), addr[:]...) +} + +// indexForNodesWithHashPrefix returns a prefix containing a key used in +// KeyNodes method. Key is hex encoded to be able to use keyTermByte +// for splitting key and node address. +func indexForNodesWithHashPrefix(key []byte) []byte { + return append([]byte{indexForHashesByNodePrefix}, append([]byte(common.Bytes2Hex(key)), keyTermByte)...) +} + +// indexForNodes constructs a database key to store keys used in +// Nodes method. +func indexForNodes(addr common.Address) []byte { + return append([]byte{indexForNodesPrefix}, addr[:]...) +} + +// indexForHashes constructs a database key to store keys used in +// Keys method. +func indexForHashes(key []byte) []byte { + return append([]byte{indexForHashesPrefix}, key...) +} + +// indexDataKey constructs a database key for key/data storage. +func indexDataKey(key []byte) []byte { + return append([]byte{indexDataPrefix}, key...) } diff --git a/swarm/storage/mock/db/db_test.go b/swarm/storage/mock/db/db_test.go index 782faaf35c..efbf942f68 100644 --- a/swarm/storage/mock/db/db_test.go +++ b/swarm/storage/mock/db/db_test.go @@ -1,5 +1,3 @@ -// +build go1.8 -// // Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // @@ -29,47 +27,49 @@ import ( // TestDBStore is running a test.MockStore tests // using test.MockStore function. func TestDBStore(t *testing.T) { - dir, err := ioutil.TempDir("", "mock_"+t.Name()) - if err != nil { - panic(err) - } - defer os.RemoveAll(dir) - - store, err := NewGlobalStore(dir) - if err != nil { - t.Fatal(err) - } - defer store.Close() + store, cleanup := newTestStore(t) + defer cleanup() test.MockStore(t, store, 100) } +// TestDBStoreListings is running test.MockStoreListings tests. +func TestDBStoreListings(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + test.MockStoreListings(t, store, 1000) +} + // TestImportExport is running a test.ImportExport tests // using test.MockStore function. func TestImportExport(t *testing.T) { - dir1, err := ioutil.TempDir("", "mock_"+t.Name()+"_exporter") - if err != nil { - panic(err) - } - defer os.RemoveAll(dir1) + store1, cleanup := newTestStore(t) + defer cleanup() - store1, err := NewGlobalStore(dir1) - if err != nil { - t.Fatal(err) - } - defer store1.Close() - - dir2, err := ioutil.TempDir("", "mock_"+t.Name()+"_importer") - if err != nil { - panic(err) - } - defer os.RemoveAll(dir2) - - store2, err := NewGlobalStore(dir2) - if err != nil { - t.Fatal(err) - } - defer store2.Close() + store2, cleanup := newTestStore(t) + defer cleanup() test.ImportExport(t, store1, store2, 100) } + +// newTestStore creates a temporary GlobalStore +// that will be closed and data deleted when +// calling returned cleanup function. +func newTestStore(t *testing.T) (s *GlobalStore, cleanup func()) { + dir, err := ioutil.TempDir("", "swarm-mock-db-") + if err != nil { + t.Fatal(err) + } + + s, err = NewGlobalStore(dir) + if err != nil { + os.RemoveAll(dir) + t.Fatal(err) + } + + return s, func() { + s.Close() + os.RemoveAll(dir) + } +} diff --git a/swarm/storage/mock/explorer/explorer.go b/swarm/storage/mock/explorer/explorer.go new file mode 100644 index 0000000000..8fffff8fde --- /dev/null +++ b/swarm/storage/mock/explorer/explorer.go @@ -0,0 +1,257 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package explorer + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/rs/cors" +) + +const jsonContentType = "application/json; charset=utf-8" + +// NewHandler constructs an http.Handler with router +// that servers requests required by chunk explorer. +// +// /api/has-key/{node}/{key} +// /api/keys?start={key}&node={node}&limit={int[0..1000]} +// /api/nodes?start={node}&key={key}&limit={int[0..1000]} +// +// Data from global store will be served and appropriate +// CORS headers will be sent if allowed origins are provided. +func NewHandler(store mock.GlobalStorer, corsOrigins []string) (handler http.Handler) { + mux := http.NewServeMux() + mux.Handle("/api/has-key/", newHasKeyHandler(store)) + mux.Handle("/api/keys", newKeysHandler(store)) + mux.Handle("/api/nodes", newNodesHandler(store)) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + jsonStatusResponse(w, http.StatusNotFound) + }) + handler = noCacheHandler(mux) + if corsOrigins != nil { + handler = cors.New(cors.Options{ + AllowedOrigins: corsOrigins, + AllowedMethods: []string{"GET"}, + MaxAge: 600, + }).Handler(handler) + } + return handler +} + +// newHasKeyHandler returns a new handler that serves +// requests for HasKey global store method. +// Possible responses are StatusResponse with +// status codes 200 or 404 if the chunk is found or not. +func newHasKeyHandler(store mock.GlobalStorer) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + addr, key, ok := parseHasKeyPath(r.URL.Path) + if !ok { + jsonStatusResponse(w, http.StatusNotFound) + return + } + found := store.HasKey(addr, key) + if !found { + jsonStatusResponse(w, http.StatusNotFound) + return + } + jsonStatusResponse(w, http.StatusOK) + } +} + +// KeysResponse is a JSON-encoded response for global store +// Keys and NodeKeys methods. +type KeysResponse struct { + Keys []string `json:"keys"` + Next string `json:"next,omitempty"` +} + +// newKeysHandler returns a new handler that serves +// requests for Key global store method. +// HTTP response body will be JSON-encoded KeysResponse. +func newKeysHandler(store mock.GlobalStorer) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + node := q.Get("node") + start, limit := listingPage(q) + + var keys mock.Keys + if node == "" { + var err error + keys, err = store.Keys(common.Hex2Bytes(start), limit) + if err != nil { + log.Error("chunk explorer: keys handler: get keys", "start", start, "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + } else { + var err error + keys, err = store.NodeKeys(common.HexToAddress(node), common.Hex2Bytes(start), limit) + if err != nil { + log.Error("chunk explorer: keys handler: get node keys", "node", node, "start", start, "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + } + ks := make([]string, len(keys.Keys)) + for i, k := range keys.Keys { + ks[i] = common.Bytes2Hex(k) + } + data, err := json.Marshal(KeysResponse{ + Keys: ks, + Next: common.Bytes2Hex(keys.Next), + }) + if err != nil { + log.Error("chunk explorer: keys handler: json marshal", "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", jsonContentType) + _, err = io.Copy(w, bytes.NewReader(data)) + if err != nil { + log.Error("chunk explorer: keys handler: write response", "err", err) + } + } +} + +// NodesResponse is a JSON-encoded response for global store +// Nodes and KeyNodes methods. +type NodesResponse struct { + Nodes []string `json:"nodes"` + Next string `json:"next,omitempty"` +} + +// newNodesHandler returns a new handler that serves +// requests for Nodes global store method. +// HTTP response body will be JSON-encoded NodesResponse. +func newNodesHandler(store mock.GlobalStorer) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + key := q.Get("key") + var start *common.Address + queryStart, limit := listingPage(q) + if queryStart != "" { + s := common.HexToAddress(queryStart) + start = &s + } + + var nodes mock.Nodes + if key == "" { + var err error + nodes, err = store.Nodes(start, limit) + if err != nil { + log.Error("chunk explorer: nodes handler: get nodes", "start", queryStart, "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + } else { + var err error + nodes, err = store.KeyNodes(common.Hex2Bytes(key), start, limit) + if err != nil { + log.Error("chunk explorer: nodes handler: get key nodes", "key", key, "start", queryStart, "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + } + ns := make([]string, len(nodes.Addrs)) + for i, n := range nodes.Addrs { + ns[i] = n.Hex() + } + var next string + if nodes.Next != nil { + next = nodes.Next.Hex() + } + data, err := json.Marshal(NodesResponse{ + Nodes: ns, + Next: next, + }) + if err != nil { + log.Error("chunk explorer: nodes handler", "err", err) + jsonStatusResponse(w, http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", jsonContentType) + _, err = io.Copy(w, bytes.NewReader(data)) + if err != nil { + log.Error("chunk explorer: nodes handler: write response", "err", err) + } + } +} + +// parseHasKeyPath extracts address and key from HTTP request +// path for HasKey route: /api/has-key/{node}/{key}. +// If ok is false, the provided path is not matched. +func parseHasKeyPath(p string) (addr common.Address, key []byte, ok bool) { + p = strings.TrimPrefix(p, "/api/has-key/") + parts := strings.SplitN(p, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return addr, nil, false + } + addr = common.HexToAddress(parts[0]) + key = common.Hex2Bytes(parts[1]) + return addr, key, true +} + +// listingPage returns start value and listing limit +// from url query values. +func listingPage(q url.Values) (start string, limit int) { + // if limit is not a valid integer (or blank string), + // ignore the error and use the returned 0 value + limit, _ = strconv.Atoi(q.Get("limit")) + return q.Get("start"), limit +} + +// StatusResponse is a standardized JSON-encoded response +// that contains information about HTTP response code +// for easier status identification. +type StatusResponse struct { + Message string `json:"message"` + Code int `json:"code"` +} + +// jsonStatusResponse writes to the response writer +// JSON-encoded StatusResponse based on the provided status code. +func jsonStatusResponse(w http.ResponseWriter, code int) { + w.Header().Set("Content-Type", jsonContentType) + w.WriteHeader(code) + err := json.NewEncoder(w).Encode(StatusResponse{ + Message: http.StatusText(code), + Code: code, + }) + if err != nil { + log.Error("chunk explorer: json status response", "err", err) + } +} + +// noCacheHandler sets required HTTP headers to prevent +// response caching at the client side. +func noCacheHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-cache, no-store, must-revalidate") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Expires", "0") + h.ServeHTTP(w, r) + }) +} diff --git a/swarm/storage/mock/explorer/explorer_test.go b/swarm/storage/mock/explorer/explorer_test.go new file mode 100644 index 0000000000..be26684266 --- /dev/null +++ b/swarm/storage/mock/explorer/explorer_test.go @@ -0,0 +1,471 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package explorer + +import ( + "encoding/binary" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sort" + "strconv" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" +) + +// TestHandler_memGlobalStore runs a set of tests +// to validate handler with mem global store. +func TestHandler_memGlobalStore(t *testing.T) { + t.Parallel() + + globalStore := mem.NewGlobalStore() + + testHandler(t, globalStore) +} + +// TestHandler_dbGlobalStore runs a set of tests +// to validate handler with database global store. +func TestHandler_dbGlobalStore(t *testing.T) { + t.Parallel() + + dir, err := ioutil.TempDir("", "swarm-mock-explorer-db-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + globalStore, err := db.NewGlobalStore(dir) + if err != nil { + t.Fatal(err) + } + defer globalStore.Close() + + testHandler(t, globalStore) +} + +// testHandler stores data distributed by node addresses +// and validates if this data is correctly retrievable +// by using the http.Handler returned by NewHandler function. +// This test covers all HTTP routes and various get parameters +// on them to check paginated results. +func testHandler(t *testing.T, globalStore mock.GlobalStorer) { + const ( + nodeCount = 350 + keyCount = 250 + keysOnNodeCount = 150 + ) + + // keys for every node + nodeKeys := make(map[string][]string) + + // a node address that is not present in global store + invalidAddr := "0x7b8b72938c254cf002c4e1e714d27e022be88d93" + + // a key that is not present in global store + invalidKey := "f9824192fb515cfb" + + for i := 1; i <= nodeCount; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + addr := common.BytesToAddress(b).Hex() + nodeKeys[addr] = make([]string, 0) + } + + for i := 1; i <= keyCount; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + + key := common.Bytes2Hex(b) + + var c int + for addr := range nodeKeys { + nodeKeys[addr] = append(nodeKeys[addr], key) + c++ + if c >= keysOnNodeCount { + break + } + } + } + + // sort keys for every node as they are expected to be + // sorted in HTTP responses + for _, keys := range nodeKeys { + sort.Strings(keys) + } + + // nodes for every key + keyNodes := make(map[string][]string) + + // construct a reverse mapping of nodes for every key + for addr, keys := range nodeKeys { + for _, key := range keys { + keyNodes[key] = append(keyNodes[key], addr) + } + } + + // sort node addresses with case insensitive sort, + // as hex letters in node addresses are in mixed caps + for _, addrs := range keyNodes { + sortCaseInsensitive(addrs) + } + + // find a key that is not stored at the address + var ( + unmatchedAddr string + unmatchedKey string + ) + for addr, keys := range nodeKeys { + for key := range keyNodes { + var found bool + for _, k := range keys { + if k == key { + found = true + break + } + } + if !found { + unmatchedAddr = addr + unmatchedKey = key + } + break + } + if unmatchedAddr != "" { + break + } + } + // check if unmatched key/address pair is found + if unmatchedAddr == "" || unmatchedKey == "" { + t.Fatalf("could not find a key that is not associated with a node") + } + + // store the data + for addr, keys := range nodeKeys { + for _, key := range keys { + err := globalStore.Put(common.HexToAddress(addr), common.Hex2Bytes(key), []byte("data")) + if err != nil { + t.Fatal(err) + } + } + } + + handler := NewHandler(globalStore, nil) + + // this subtest confirms that it has uploaded key and that it does not have invalid keys + t.Run("has key", func(t *testing.T) { + for addr, keys := range nodeKeys { + for _, key := range keys { + testStatusResponse(t, handler, "/api/has-key/"+addr+"/"+key, http.StatusOK) + testStatusResponse(t, handler, "/api/has-key/"+invalidAddr+"/"+key, http.StatusNotFound) + } + testStatusResponse(t, handler, "/api/has-key/"+addr+"/"+invalidKey, http.StatusNotFound) + } + testStatusResponse(t, handler, "/api/has-key/"+invalidAddr+"/"+invalidKey, http.StatusNotFound) + testStatusResponse(t, handler, "/api/has-key/"+unmatchedAddr+"/"+unmatchedKey, http.StatusNotFound) + }) + + // this subtest confirms that all keys are are listed in correct order with expected pagination + t.Run("keys", func(t *testing.T) { + var allKeys []string + for key := range keyNodes { + allKeys = append(allKeys, key) + } + sort.Strings(allKeys) + + t.Run("limit 0", testKeys(handler, allKeys, 0, "")) + t.Run("limit default", testKeys(handler, allKeys, mock.DefaultLimit, "")) + t.Run("limit 2x default", testKeys(handler, allKeys, 2*mock.DefaultLimit, "")) + t.Run("limit 0.5x default", testKeys(handler, allKeys, mock.DefaultLimit/2, "")) + t.Run("limit max", testKeys(handler, allKeys, mock.MaxLimit, "")) + t.Run("limit 2x max", testKeys(handler, allKeys, 2*mock.MaxLimit, "")) + t.Run("limit negative", testKeys(handler, allKeys, -10, "")) + }) + + // this subtest confirms that all keys are are listed for every node in correct order + // and that for one node different pagination options are correct + t.Run("node keys", func(t *testing.T) { + var limitCheckAddr string + + for addr, keys := range nodeKeys { + testKeys(handler, keys, 0, addr)(t) + if limitCheckAddr == "" { + limitCheckAddr = addr + } + } + testKeys(handler, nil, 0, invalidAddr)(t) + + limitCheckKeys := nodeKeys[limitCheckAddr] + t.Run("limit 0", testKeys(handler, limitCheckKeys, 0, limitCheckAddr)) + t.Run("limit default", testKeys(handler, limitCheckKeys, mock.DefaultLimit, limitCheckAddr)) + t.Run("limit 2x default", testKeys(handler, limitCheckKeys, 2*mock.DefaultLimit, limitCheckAddr)) + t.Run("limit 0.5x default", testKeys(handler, limitCheckKeys, mock.DefaultLimit/2, limitCheckAddr)) + t.Run("limit max", testKeys(handler, limitCheckKeys, mock.MaxLimit, limitCheckAddr)) + t.Run("limit 2x max", testKeys(handler, limitCheckKeys, 2*mock.MaxLimit, limitCheckAddr)) + t.Run("limit negative", testKeys(handler, limitCheckKeys, -10, limitCheckAddr)) + }) + + // this subtest confirms that all nodes are are listed in correct order with expected pagination + t.Run("nodes", func(t *testing.T) { + var allNodes []string + for addr := range nodeKeys { + allNodes = append(allNodes, addr) + } + sortCaseInsensitive(allNodes) + + t.Run("limit 0", testNodes(handler, allNodes, 0, "")) + t.Run("limit default", testNodes(handler, allNodes, mock.DefaultLimit, "")) + t.Run("limit 2x default", testNodes(handler, allNodes, 2*mock.DefaultLimit, "")) + t.Run("limit 0.5x default", testNodes(handler, allNodes, mock.DefaultLimit/2, "")) + t.Run("limit max", testNodes(handler, allNodes, mock.MaxLimit, "")) + t.Run("limit 2x max", testNodes(handler, allNodes, 2*mock.MaxLimit, "")) + t.Run("limit negative", testNodes(handler, allNodes, -10, "")) + }) + + // this subtest confirms that all nodes are are listed that contain a a particular key in correct order + // and that for one key different node pagination options are correct + t.Run("key nodes", func(t *testing.T) { + var limitCheckKey string + + for key, addrs := range keyNodes { + testNodes(handler, addrs, 0, key)(t) + if limitCheckKey == "" { + limitCheckKey = key + } + } + testNodes(handler, nil, 0, invalidKey)(t) + + limitCheckKeys := keyNodes[limitCheckKey] + t.Run("limit 0", testNodes(handler, limitCheckKeys, 0, limitCheckKey)) + t.Run("limit default", testNodes(handler, limitCheckKeys, mock.DefaultLimit, limitCheckKey)) + t.Run("limit 2x default", testNodes(handler, limitCheckKeys, 2*mock.DefaultLimit, limitCheckKey)) + t.Run("limit 0.5x default", testNodes(handler, limitCheckKeys, mock.DefaultLimit/2, limitCheckKey)) + t.Run("limit max", testNodes(handler, limitCheckKeys, mock.MaxLimit, limitCheckKey)) + t.Run("limit 2x max", testNodes(handler, limitCheckKeys, 2*mock.MaxLimit, limitCheckKey)) + t.Run("limit negative", testNodes(handler, limitCheckKeys, -10, limitCheckKey)) + }) +} + +// testsKeys returns a test function that validates wantKeys against a series of /api/keys +// HTTP responses with provided limit and node options. +func testKeys(handler http.Handler, wantKeys []string, limit int, node string) func(t *testing.T) { + return func(t *testing.T) { + t.Helper() + + wantLimit := limit + if wantLimit <= 0 { + wantLimit = mock.DefaultLimit + } + if wantLimit > mock.MaxLimit { + wantLimit = mock.MaxLimit + } + wantKeysLen := len(wantKeys) + var i int + var startKey string + for { + var wantNext string + start := i * wantLimit + end := (i + 1) * wantLimit + if end < wantKeysLen { + wantNext = wantKeys[end] + } else { + end = wantKeysLen + } + testKeysResponse(t, handler, node, startKey, limit, KeysResponse{ + Keys: wantKeys[start:end], + Next: wantNext, + }) + if wantNext == "" { + break + } + startKey = wantNext + i++ + } + } +} + +// testNodes returns a test function that validates wantAddrs against a series of /api/nodes +// HTTP responses with provided limit and key options. +func testNodes(handler http.Handler, wantAddrs []string, limit int, key string) func(t *testing.T) { + return func(t *testing.T) { + t.Helper() + + wantLimit := limit + if wantLimit <= 0 { + wantLimit = mock.DefaultLimit + } + if wantLimit > mock.MaxLimit { + wantLimit = mock.MaxLimit + } + wantAddrsLen := len(wantAddrs) + var i int + var startKey string + for { + var wantNext string + start := i * wantLimit + end := (i + 1) * wantLimit + if end < wantAddrsLen { + wantNext = wantAddrs[end] + } else { + end = wantAddrsLen + } + testNodesResponse(t, handler, key, startKey, limit, NodesResponse{ + Nodes: wantAddrs[start:end], + Next: wantNext, + }) + if wantNext == "" { + break + } + startKey = wantNext + i++ + } + } +} + +// testStatusResponse validates a response made on url if it matches +// the expected StatusResponse. +func testStatusResponse(t *testing.T, handler http.Handler, url string, code int) { + t.Helper() + + resp := httpGet(t, handler, url) + + if resp.StatusCode != code { + t.Errorf("got status code %v, want %v", resp.StatusCode, code) + } + if got := resp.Header.Get("Content-Type"); got != jsonContentType { + t.Errorf("got Content-Type header %q, want %q", got, jsonContentType) + } + var r StatusResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if r.Code != code { + t.Errorf("got response code %v, want %v", r.Code, code) + } + if r.Message != http.StatusText(code) { + t.Errorf("got response message %q, want %q", r.Message, http.StatusText(code)) + } +} + +// testKeysResponse validates response returned from handler on /api/keys +// with node, start and limit options against KeysResponse. +func testKeysResponse(t *testing.T, handler http.Handler, node, start string, limit int, want KeysResponse) { + t.Helper() + + u, err := url.Parse("/api/keys") + if err != nil { + t.Fatal(err) + } + q := u.Query() + if node != "" { + q.Set("node", node) + } + if start != "" { + q.Set("start", start) + } + if limit != 0 { + q.Set("limit", strconv.Itoa(limit)) + } + u.RawQuery = q.Encode() + + resp := httpGet(t, handler, u.String()) + + if resp.StatusCode != http.StatusOK { + t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Content-Type"); got != jsonContentType { + t.Errorf("got Content-Type header %q, want %q", got, jsonContentType) + } + var r KeysResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if fmt.Sprint(r.Keys) != fmt.Sprint(want.Keys) { + t.Errorf("got keys %v, want %v", r.Keys, want.Keys) + } + if r.Next != want.Next { + t.Errorf("got next %s, want %s", r.Next, want.Next) + } +} + +// testNodesResponse validates response returned from handler on /api/nodes +// with key, start and limit options against NodesResponse. +func testNodesResponse(t *testing.T, handler http.Handler, key, start string, limit int, want NodesResponse) { + t.Helper() + + u, err := url.Parse("/api/nodes") + if err != nil { + t.Fatal(err) + } + q := u.Query() + if key != "" { + q.Set("key", key) + } + if start != "" { + q.Set("start", start) + } + if limit != 0 { + q.Set("limit", strconv.Itoa(limit)) + } + u.RawQuery = q.Encode() + + resp := httpGet(t, handler, u.String()) + + if resp.StatusCode != http.StatusOK { + t.Errorf("got status code %v, want %v", resp.StatusCode, http.StatusOK) + } + if got := resp.Header.Get("Content-Type"); got != jsonContentType { + t.Errorf("got Content-Type header %q, want %q", got, jsonContentType) + } + var r NodesResponse + if err := json.NewDecoder(resp.Body).Decode(&r); err != nil { + t.Fatal(err) + } + if fmt.Sprint(r.Nodes) != fmt.Sprint(want.Nodes) { + t.Errorf("got nodes %v, want %v", r.Nodes, want.Nodes) + } + if r.Next != want.Next { + t.Errorf("got next %s, want %s", r.Next, want.Next) + } +} + +// httpGet uses httptest recorder to provide a response on handler's url. +func httpGet(t *testing.T, handler http.Handler, url string) (r *http.Response) { + t.Helper() + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + t.Fatal(err) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + return w.Result() +} + +// sortCaseInsensitive performs a case insensitive sort on a string slice. +func sortCaseInsensitive(s []string) { + sort.Slice(s, func(i, j int) bool { + return strings.ToLower(s[i]) < strings.ToLower(s[j]) + }) +} diff --git a/swarm/storage/mock/explorer/headers_test.go b/swarm/storage/mock/explorer/headers_test.go new file mode 100644 index 0000000000..5b8e05ffde --- /dev/null +++ b/swarm/storage/mock/explorer/headers_test.go @@ -0,0 +1,163 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package explorer + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" +) + +// TestHandler_CORSOrigin validates that the correct Access-Control-Allow-Origin +// header is served with various allowed origin settings. +func TestHandler_CORSOrigin(t *testing.T) { + notAllowedOrigin := "http://not-allowed-origin.com/" + + for _, tc := range []struct { + name string + origins []string + }{ + { + name: "no origin", + origins: nil, + }, + { + name: "single origin", + origins: []string{"http://localhost/"}, + }, + { + name: "multiple origins", + origins: []string{"http://localhost/", "http://ethereum.org/"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + handler := NewHandler(mem.NewGlobalStore(), tc.origins) + + origins := tc.origins + if origins == nil { + // handle the "no origin" test case + origins = []string{""} + } + + for _, origin := range origins { + t.Run(fmt.Sprintf("get %q", origin), newTestCORSOrigin(handler, origin, origin)) + t.Run(fmt.Sprintf("preflight %q", origin), newTestCORSPreflight(handler, origin, origin)) + } + + t.Run(fmt.Sprintf("get %q", notAllowedOrigin), newTestCORSOrigin(handler, notAllowedOrigin, "")) + t.Run(fmt.Sprintf("preflight %q", notAllowedOrigin), newTestCORSPreflight(handler, notAllowedOrigin, "")) + }) + } + + t.Run("wildcard", func(t *testing.T) { + handler := NewHandler(mem.NewGlobalStore(), []string{"*"}) + + for _, origin := range []string{ + "http://example.com/", + "http://ethereum.org", + "http://localhost", + } { + t.Run(fmt.Sprintf("get %q", origin), newTestCORSOrigin(handler, origin, origin)) + t.Run(fmt.Sprintf("preflight %q", origin), newTestCORSPreflight(handler, origin, origin)) + } + }) +} + +// newTestCORSOrigin returns a test function that validates if wantOrigin CORS header is +// served by the handler for a GET request. +func newTestCORSOrigin(handler http.Handler, origin, wantOrigin string) func(t *testing.T) { + return func(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "/", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", origin) + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + resp := w.Result() + + header := resp.Header.Get("Access-Control-Allow-Origin") + if header != wantOrigin { + t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, wantOrigin) + } + } +} + +// newTestCORSPreflight returns a test function that validates if wantOrigin CORS header is +// served by the handler for an OPTIONS CORS preflight request. +func newTestCORSPreflight(handler http.Handler, origin, wantOrigin string) func(t *testing.T) { + return func(t *testing.T) { + req, err := http.NewRequest(http.MethodOptions, "/", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Origin", origin) + req.Header.Set("Access-Control-Request-Method", "GET") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + resp := w.Result() + + header := resp.Header.Get("Access-Control-Allow-Origin") + if header != wantOrigin { + t.Errorf("got Access-Control-Allow-Origin header %q, want %q", header, wantOrigin) + } + } +} + +// TestHandler_noCacheHeaders validates that no cache headers are server. +func TestHandler_noCacheHeaders(t *testing.T) { + handler := NewHandler(mem.NewGlobalStore(), nil) + + for _, tc := range []struct { + url string + }{ + { + url: "/", + }, + { + url: "/api/nodes", + }, + { + url: "/api/keys", + }, + } { + req, err := http.NewRequest(http.MethodGet, tc.url, nil) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, req) + resp := w.Result() + + for header, want := range map[string]string{ + "Cache-Control": "no-cache, no-store, must-revalidate", + "Pragma": "no-cache", + "Expires": "0", + } { + got := resp.Header.Get(header) + if got != want { + t.Errorf("got %q header %q for url %q, want %q", header, tc.url, got, want) + } + } + } +} diff --git a/swarm/storage/mock/explorer/swagger.yaml b/swarm/storage/mock/explorer/swagger.yaml new file mode 100644 index 0000000000..2c014e927e --- /dev/null +++ b/swarm/storage/mock/explorer/swagger.yaml @@ -0,0 +1,176 @@ +swagger: '2.0' +info: + title: Swarm Global Store API + version: 0.1.0 +tags: + - name: Has Key + description: Checks if a Key is stored on a Node + - name: Keys + description: Lists Keys + - name: Nodes + description: Lists Node addresses + +paths: + '/api/has-key/{node}/{key}': + get: + tags: + - Has Key + summary: Checks if a Key is stored on a Node + operationId: hasKey + produces: + - application/json + + parameters: + - name: node + in: path + required: true + type: string + format: hex-endoded + description: Node address. + + - name: key + in: path + required: true + type: string + format: hex-endoded + description: Key. + + responses: + '200': + description: Key is stored on Node + schema: + $ref: '#/definitions/Status' + '404': + description: Key is not stored on Node + schema: + $ref: '#/definitions/Status' + '500': + description: Internal Server Error + schema: + $ref: '#/definitions/Status' + + '/api/keys': + get: + tags: + - Keys + summary: Lists Keys + operationId: keys + produces: + - application/json + + parameters: + - name: start + in: query + required: false + type: string + format: hex-encoded Key + description: A Key as the starting point for the returned list. It is usually a value from the returned "next" field in the Keys repsonse. + + - name: limit + in: query + required: false + type: integer + default: 100 + minimum: 1 + maximum: 1000 + description: Limits the number of Keys returned in on response. + + - name: node + in: query + required: false + type: string + format: hex-encoded Node address + description: If this parameter is provided, only Keys that are stored on this Node be returned in the response. If not, all known Keys will be returned. + + responses: + '200': + description: List of Keys + schema: + $ref: '#/definitions/Keys' + '500': + description: Internal Server Error + schema: + $ref: '#/definitions/Status' + + '/api/nodes': + get: + tags: + - Nodes + summary: Lists Node addresses + operationId: nodes + produces: + - application/json + + parameters: + - name: start + in: query + required: false + type: string + format: hex-encoded Node address + description: A Node address as the starting point for the returned list. It is usually a value from the returned "next" field in the Nodes repsonse. + + - name: limit + in: query + required: false + type: integer + default: 100 + minimum: 1 + maximum: 1000 + description: Limits the number of Node addresses returned in on response. + + - name: key + in: query + required: false + type: string + format: hex-encoded Key + description: If this parameter is provided, only addresses of Nodes that store this Key will be returned in the response. If not, all known Node addresses will be returned. + + responses: + '200': + description: List of Node addresses + schema: + $ref: '#/definitions/Nodes' + '500': + description: Internal Server Error + schema: + $ref: '#/definitions/Status' + +definitions: + + Status: + type: object + properties: + message: + type: string + description: HTTP Status Code name. + code: + type: integer + description: HTTP Status Code. + + Keys: + type: object + properties: + keys: + type: array + description: A list of Keys. + items: + type: string + format: hex-encoded Key + next: + type: string + format: hex-encoded Key + description: If present, the next Key in listing. Can be passed as "start" query parameter to continue the listing. If not present, the end of the listing is reached. + + Nodes: + type: object + properties: + nodes: + type: array + description: A list of Node addresses. + items: + type: string + format: hex-encoded Node address + next: + type: string + format: hex-encoded Node address + description: If present, the next Node address in listing. Can be passed as "start" query parameter to continue the listing. If not present, the end of the listing is reached. diff --git a/swarm/storage/mock/mem/mem.go b/swarm/storage/mock/mem/mem.go index 3a0a2beb8d..38bf098df4 100644 --- a/swarm/storage/mock/mem/mem.go +++ b/swarm/storage/mock/mem/mem.go @@ -25,6 +25,7 @@ import ( "encoding/json" "io" "io/ioutil" + "sort" "sync" "github.com/ethereum/go-ethereum/common" @@ -34,16 +35,27 @@ import ( // GlobalStore stores all chunk data and also keys and node addresses relations. // It implements mock.GlobalStore interface. type GlobalStore struct { - nodes map[string]map[common.Address]struct{} - data map[string][]byte - mu sync.Mutex + // holds a slice of keys per node + nodeKeys map[common.Address][][]byte + // holds which key is stored on which nodes + keyNodes map[string][]common.Address + // all node addresses + nodes []common.Address + // all keys + keys [][]byte + // all keys data + data map[string][]byte + mu sync.RWMutex } // NewGlobalStore creates a new instance of GlobalStore. func NewGlobalStore() *GlobalStore { return &GlobalStore{ - nodes: make(map[string]map[common.Address]struct{}), - data: make(map[string][]byte), + nodeKeys: make(map[common.Address][][]byte), + keyNodes: make(map[string][]common.Address), + nodes: make([]common.Address, 0), + keys: make([][]byte, 0), + data: make(map[string][]byte), } } @@ -56,10 +68,10 @@ func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore { // Get returns chunk data if the chunk with key exists for node // on address addr. func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) { - s.mu.Lock() - defer s.mu.Unlock() + s.mu.RLock() + defer s.mu.RUnlock() - if _, ok := s.nodes[string(key)][addr]; !ok { + if _, has := s.nodeKeyIndex(addr, key); !has { return nil, mock.ErrNotFound } @@ -75,11 +87,33 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error { s.mu.Lock() defer s.mu.Unlock() - if _, ok := s.nodes[string(key)]; !ok { - s.nodes[string(key)] = make(map[common.Address]struct{}) + if i, found := s.nodeKeyIndex(addr, key); !found { + s.nodeKeys[addr] = append(s.nodeKeys[addr], nil) + copy(s.nodeKeys[addr][i+1:], s.nodeKeys[addr][i:]) + s.nodeKeys[addr][i] = key } - s.nodes[string(key)][addr] = struct{}{} + + if i, found := s.keyNodeIndex(key, addr); !found { + k := string(key) + s.keyNodes[k] = append(s.keyNodes[k], addr) + copy(s.keyNodes[k][i+1:], s.keyNodes[k][i:]) + s.keyNodes[k][i] = addr + } + + if i, found := s.nodeIndex(addr); !found { + s.nodes = append(s.nodes, addr) + copy(s.nodes[i+1:], s.nodes[i:]) + s.nodes[i] = addr + } + + if i, found := s.keyIndex(key); !found { + s.keys = append(s.keys, nil) + copy(s.keys[i+1:], s.keys[i:]) + s.keys[i] = key + } + s.data[string(key)] = data + return nil } @@ -88,24 +122,177 @@ func (s *GlobalStore) Delete(addr common.Address, key []byte) error { s.mu.Lock() defer s.mu.Unlock() - var count int - if _, ok := s.nodes[string(key)]; ok { - delete(s.nodes[string(key)], addr) - count = len(s.nodes[string(key)]) + if i, has := s.nodeKeyIndex(addr, key); has { + s.nodeKeys[addr] = append(s.nodeKeys[addr][:i], s.nodeKeys[addr][i+1:]...) } - if count == 0 { - delete(s.data, string(key)) + + k := string(key) + if i, on := s.keyNodeIndex(key, addr); on { + s.keyNodes[k] = append(s.keyNodes[k][:i], s.keyNodes[k][i+1:]...) + } + + if len(s.nodeKeys[addr]) == 0 { + if i, found := s.nodeIndex(addr); found { + s.nodes = append(s.nodes[:i], s.nodes[i+1:]...) + } + } + + if len(s.keyNodes[k]) == 0 { + if i, found := s.keyIndex(key); found { + s.keys = append(s.keys[:i], s.keys[i+1:]...) + } } return nil } // HasKey returns whether a node with addr contains the key. -func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { - s.mu.Lock() - defer s.mu.Unlock() +func (s *GlobalStore) HasKey(addr common.Address, key []byte) (yes bool) { + s.mu.RLock() + defer s.mu.RUnlock() - _, ok := s.nodes[string(key)][addr] - return ok + _, yes = s.nodeKeyIndex(addr, key) + return yes +} + +// keyIndex returns the index of a key in keys slice. +func (s *GlobalStore) keyIndex(key []byte) (index int, found bool) { + l := len(s.keys) + index = sort.Search(l, func(i int) bool { + return bytes.Compare(s.keys[i], key) >= 0 + }) + found = index < l && bytes.Equal(s.keys[index], key) + return index, found +} + +// nodeIndex returns the index of a node address in nodes slice. +func (s *GlobalStore) nodeIndex(addr common.Address) (index int, found bool) { + l := len(s.nodes) + index = sort.Search(l, func(i int) bool { + return bytes.Compare(s.nodes[i][:], addr[:]) >= 0 + }) + found = index < l && bytes.Equal(s.nodes[index][:], addr[:]) + return index, found +} + +// nodeKeyIndex returns the index of a key in nodeKeys slice. +func (s *GlobalStore) nodeKeyIndex(addr common.Address, key []byte) (index int, found bool) { + l := len(s.nodeKeys[addr]) + index = sort.Search(l, func(i int) bool { + return bytes.Compare(s.nodeKeys[addr][i], key) >= 0 + }) + found = index < l && bytes.Equal(s.nodeKeys[addr][index], key) + return index, found +} + +// keyNodeIndex returns the index of a node address in keyNodes slice. +func (s *GlobalStore) keyNodeIndex(key []byte, addr common.Address) (index int, found bool) { + k := string(key) + l := len(s.keyNodes[k]) + index = sort.Search(l, func(i int) bool { + return bytes.Compare(s.keyNodes[k][i][:], addr[:]) >= 0 + }) + found = index < l && s.keyNodes[k][index] == addr + return index, found +} + +// Keys returns a paginated list of keys on all nodes. +func (s *GlobalStore) Keys(startKey []byte, limit int) (keys mock.Keys, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + var i int + if startKey != nil { + i, _ = s.keyIndex(startKey) + } + total := len(s.keys) + max := maxIndex(i, limit, total) + keys.Keys = make([][]byte, 0, max-i) + for ; i < max; i++ { + keys.Keys = append(keys.Keys, append([]byte(nil), s.keys[i]...)) + } + if total > max { + keys.Next = s.keys[max] + } + return keys, nil +} + +// Nodes returns a paginated list of all known nodes. +func (s *GlobalStore) Nodes(startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + var i int + if startAddr != nil { + i, _ = s.nodeIndex(*startAddr) + } + total := len(s.nodes) + max := maxIndex(i, limit, total) + nodes.Addrs = make([]common.Address, 0, max-i) + for ; i < max; i++ { + nodes.Addrs = append(nodes.Addrs, s.nodes[i]) + } + if total > max { + nodes.Next = &s.nodes[max] + } + return nodes, nil +} + +// NodeKeys returns a paginated list of keys on a node with provided address. +func (s *GlobalStore) NodeKeys(addr common.Address, startKey []byte, limit int) (keys mock.Keys, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + var i int + if startKey != nil { + i, _ = s.nodeKeyIndex(addr, startKey) + } + total := len(s.nodeKeys[addr]) + max := maxIndex(i, limit, total) + keys.Keys = make([][]byte, 0, max-i) + for ; i < max; i++ { + keys.Keys = append(keys.Keys, append([]byte(nil), s.nodeKeys[addr][i]...)) + } + if total > max { + keys.Next = s.nodeKeys[addr][max] + } + return keys, nil +} + +// KeyNodes returns a paginated list of nodes that contain a particular key. +func (s *GlobalStore) KeyNodes(key []byte, startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + s.mu.RLock() + defer s.mu.RUnlock() + + var i int + if startAddr != nil { + i, _ = s.keyNodeIndex(key, *startAddr) + } + total := len(s.keyNodes[string(key)]) + max := maxIndex(i, limit, total) + nodes.Addrs = make([]common.Address, 0, max-i) + for ; i < max; i++ { + nodes.Addrs = append(nodes.Addrs, s.keyNodes[string(key)][i]) + } + if total > max { + nodes.Next = &s.keyNodes[string(key)][max] + } + return nodes, nil +} + +// maxIndex returns the end index for one page listing +// based on the start index, limit and total number of elements. +func maxIndex(start, limit, total int) (max int) { + if limit <= 0 { + limit = mock.DefaultLimit + } + if limit > mock.MaxLimit { + limit = mock.MaxLimit + } + max = total + if start+limit < max { + max = start + limit + } + return max } // Import reads tar archive from a reader that contains exported chunk data. @@ -135,14 +322,26 @@ func (s *GlobalStore) Import(r io.Reader) (n int, err error) { return n, err } - addrs := make(map[common.Address]struct{}) - for _, a := range c.Addrs { - addrs[a] = struct{}{} + key := common.Hex2Bytes(hdr.Name) + s.keyNodes[string(key)] = c.Addrs + for _, addr := range c.Addrs { + if i, has := s.nodeKeyIndex(addr, key); !has { + s.nodeKeys[addr] = append(s.nodeKeys[addr], nil) + copy(s.nodeKeys[addr][i+1:], s.nodeKeys[addr][i:]) + s.nodeKeys[addr][i] = key + } + if i, found := s.nodeIndex(addr); !found { + s.nodes = append(s.nodes, addr) + copy(s.nodes[i+1:], s.nodes[i:]) + s.nodes[i] = addr + } } - - key := string(common.Hex2Bytes(hdr.Name)) - s.nodes[key] = addrs - s.data[key] = c.Data + if i, found := s.keyIndex(key); !found { + s.keys = append(s.keys, nil) + copy(s.keys[i+1:], s.keys[i:]) + s.keys[i] = key + } + s.data[string(key)] = c.Data n++ } return n, err @@ -151,23 +350,18 @@ func (s *GlobalStore) Import(r io.Reader) (n int, err error) { // Export writes to a writer a tar archive with all chunk data from // the store. It returns the number of chunks exported and an error. func (s *GlobalStore) Export(w io.Writer) (n int, err error) { - s.mu.Lock() - defer s.mu.Unlock() + s.mu.RLock() + defer s.mu.RUnlock() tw := tar.NewWriter(w) defer tw.Close() buf := bytes.NewBuffer(make([]byte, 0, 1024)) encoder := json.NewEncoder(buf) - for key, addrs := range s.nodes { - al := make([]common.Address, 0, len(addrs)) - for a := range addrs { - al = append(al, a) - } - + for key, addrs := range s.keyNodes { buf.Reset() if err = encoder.Encode(mock.ExportedChunk{ - Addrs: al, + Addrs: addrs, Data: s.data[key], }); err != nil { return n, err diff --git a/swarm/storage/mock/mem/mem_test.go b/swarm/storage/mock/mem/mem_test.go index adcefaabb4..d39aaef454 100644 --- a/swarm/storage/mock/mem/mem_test.go +++ b/swarm/storage/mock/mem/mem_test.go @@ -28,6 +28,12 @@ func TestGlobalStore(t *testing.T) { test.MockStore(t, NewGlobalStore(), 100) } +// TestGlobalStoreListings is running test for a GlobalStore +// using test.MockStoreListings function. +func TestGlobalStoreListings(t *testing.T) { + test.MockStoreListings(t, NewGlobalStore(), 1000) +} + // TestImportExport is running tests for importing and // exporting data between two GlobalStores // using test.ImportExport function. diff --git a/swarm/storage/mock/mock.go b/swarm/storage/mock/mock.go index 626ba3fe1b..586112a98b 100644 --- a/swarm/storage/mock/mock.go +++ b/swarm/storage/mock/mock.go @@ -39,6 +39,17 @@ import ( "github.com/ethereum/go-ethereum/common" ) +const ( + // DefaultLimit should be used as default limit for + // Keys, Nodes, NodeKeys and KeyNodes GlobarStorer + // methids implementations. + DefaultLimit = 100 + // MaxLimit should be used as the maximal returned number + // of items for Keys, Nodes, NodeKeys and KeyNodes GlobarStorer + // methids implementations, regardless of provided limit. + MaxLimit = 1000 +) + // ErrNotFound indicates that the chunk is not found. var ErrNotFound = errors.New("not found") @@ -76,6 +87,10 @@ func (n *NodeStore) Delete(key []byte) error { return n.store.Delete(n.addr, key) } +func (n *NodeStore) Keys(startKey []byte, limit int) (keys Keys, err error) { + return n.store.NodeKeys(n.addr, startKey, limit) +} + // GlobalStorer defines methods for mock db store // that stores chunk data for all swarm nodes. // It is used in tests to construct mock NodeStores @@ -85,12 +100,28 @@ type GlobalStorer interface { Put(addr common.Address, key []byte, data []byte) error Delete(addr common.Address, key []byte) error HasKey(addr common.Address, key []byte) bool + Keys(startKey []byte, limit int) (keys Keys, err error) + Nodes(startAddr *common.Address, limit int) (nodes Nodes, err error) + NodeKeys(addr common.Address, startKey []byte, limit int) (keys Keys, err error) + KeyNodes(key []byte, startAddr *common.Address, limit int) (nodes Nodes, err error) // NewNodeStore creates an instance of NodeStore // to be used by a single swarm node with // address addr. NewNodeStore(addr common.Address) *NodeStore } +// Keys are returned results by Keys and NodeKeys GlobalStorer methods. +type Keys struct { + Keys [][]byte + Next []byte +} + +// Nodes are returned results by Nodes and KeyNodes GlobalStorer methods. +type Nodes struct { + Addrs []common.Address + Next *common.Address +} + // Importer defines method for importing mock store data // from an exported tar archive. type Importer interface { diff --git a/swarm/storage/mock/rpc/rpc.go b/swarm/storage/mock/rpc/rpc.go index 8cd6c83a7a..8150ccff13 100644 --- a/swarm/storage/mock/rpc/rpc.go +++ b/swarm/storage/mock/rpc/rpc.go @@ -88,3 +88,27 @@ func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool { } return has } + +// Keys returns a paginated list of keys on all nodes. +func (s *GlobalStore) Keys(startKey []byte, limit int) (keys mock.Keys, err error) { + err = s.client.Call(&keys, "mockStore_keys", startKey, limit) + return keys, err +} + +// Nodes returns a paginated list of all known nodes. +func (s *GlobalStore) Nodes(startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + err = s.client.Call(&nodes, "mockStore_nodes", startAddr, limit) + return nodes, err +} + +// NodeKeys returns a paginated list of keys on a node with provided address. +func (s *GlobalStore) NodeKeys(addr common.Address, startKey []byte, limit int) (keys mock.Keys, err error) { + err = s.client.Call(&keys, "mockStore_nodeKeys", addr, startKey, limit) + return keys, err +} + +// KeyNodes returns a paginated list of nodes that contain a particular key. +func (s *GlobalStore) KeyNodes(key []byte, startAddr *common.Address, limit int) (nodes mock.Nodes, err error) { + err = s.client.Call(&nodes, "mockStore_keyNodes", key, startAddr, limit) + return nodes, err +} diff --git a/swarm/storage/mock/rpc/rpc_test.go b/swarm/storage/mock/rpc/rpc_test.go index f62340edec..6c46523558 100644 --- a/swarm/storage/mock/rpc/rpc_test.go +++ b/swarm/storage/mock/rpc/rpc_test.go @@ -27,6 +27,27 @@ import ( // TestDBStore is running test for a GlobalStore // using test.MockStore function. func TestRPCStore(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + test.MockStore(t, store, 30) +} + +// TestRPCStoreListings is running test for a GlobalStore +// using test.MockStoreListings function. +func TestRPCStoreListings(t *testing.T) { + store, cleanup := newTestStore(t) + defer cleanup() + + test.MockStoreListings(t, store, 1000) +} + +// newTestStore creates a temporary GlobalStore +// that will be closed when returned cleanup function +// is called. +func newTestStore(t *testing.T) (s *GlobalStore, cleanup func()) { + t.Helper() + serverStore := mem.NewGlobalStore() server := rpc.NewServer() @@ -35,7 +56,9 @@ func TestRPCStore(t *testing.T) { } store := NewGlobalStore(rpc.DialInProc(server)) - defer store.Close() - - test.MockStore(t, store, 30) + return store, func() { + if err := store.Close(); err != nil { + t.Error(err) + } + } } diff --git a/swarm/storage/mock/test/test.go b/swarm/storage/mock/test/test.go index 69828b144f..cc837f0b77 100644 --- a/swarm/storage/mock/test/test.go +++ b/swarm/storage/mock/test/test.go @@ -20,6 +20,7 @@ package test import ( "bytes" + "encoding/binary" "fmt" "io" "strconv" @@ -170,6 +171,123 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) { }) } +// MockStoreListings tests global store methods Keys, Nodes, NodeKeys and KeyNodes. +// It uses a provided globalstore to put chunks for n number of node addresses +// and to validate that methods are returning the right responses. +func MockStoreListings(t *testing.T, globalStore mock.GlobalStorer, n int) { + addrs := make([]common.Address, n) + for i := 0; i < n; i++ { + addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16)) + } + type chunk struct { + key []byte + data []byte + } + const chunksPerNode = 5 + keys := make([][]byte, n*chunksPerNode) + for i := 0; i < n*chunksPerNode; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + keys[i] = b + } + + // keep track of keys on every node + nodeKeys := make(map[common.Address][][]byte) + // keep track of nodes that store particular key + keyNodes := make(map[string][]common.Address) + for i := 0; i < chunksPerNode; i++ { + // put chunks for every address + for j := 0; j < n; j++ { + addr := addrs[j] + key := keys[(i*n)+j] + err := globalStore.Put(addr, key, []byte("data")) + if err != nil { + t.Fatal(err) + } + nodeKeys[addr] = append(nodeKeys[addr], key) + keyNodes[string(key)] = append(keyNodes[string(key)], addr) + } + + // test Keys method + var startKey []byte + var gotKeys [][]byte + for { + keys, err := globalStore.Keys(startKey, 0) + if err != nil { + t.Fatal(err) + } + gotKeys = append(gotKeys, keys.Keys...) + if keys.Next == nil { + break + } + startKey = keys.Next + } + wantKeys := keys[:(i+1)*n] + if fmt.Sprint(gotKeys) != fmt.Sprint(wantKeys) { + t.Fatalf("got #%v keys %v, want %v", i+1, gotKeys, wantKeys) + } + + // test Nodes method + var startNode *common.Address + var gotNodes []common.Address + for { + nodes, err := globalStore.Nodes(startNode, 0) + if err != nil { + t.Fatal(err) + } + gotNodes = append(gotNodes, nodes.Addrs...) + if nodes.Next == nil { + break + } + startNode = nodes.Next + } + wantNodes := addrs + if fmt.Sprint(gotNodes) != fmt.Sprint(wantNodes) { + t.Fatalf("got #%v nodes %v, want %v", i+1, gotNodes, wantNodes) + } + + // test NodeKeys method + for addr, wantKeys := range nodeKeys { + var startKey []byte + var gotKeys [][]byte + for { + keys, err := globalStore.NodeKeys(addr, startKey, 0) + if err != nil { + t.Fatal(err) + } + gotKeys = append(gotKeys, keys.Keys...) + if keys.Next == nil { + break + } + startKey = keys.Next + } + if fmt.Sprint(gotKeys) != fmt.Sprint(wantKeys) { + t.Fatalf("got #%v %s node keys %v, want %v", i+1, addr.Hex(), gotKeys, wantKeys) + } + } + + // test KeyNodes method + for key, wantNodes := range keyNodes { + var startNode *common.Address + var gotNodes []common.Address + for { + nodes, err := globalStore.KeyNodes([]byte(key), startNode, 0) + if err != nil { + t.Fatal(err) + } + gotNodes = append(gotNodes, nodes.Addrs...) + if nodes.Next == nil { + break + } + startNode = nodes.Next + } + if fmt.Sprint(gotNodes) != fmt.Sprint(wantNodes) { + t.Fatalf("got #%v %x key nodes %v, want %v", i+1, []byte(key), gotNodes, wantNodes) + } + } + } +} + // ImportExport saves chunks to the outStore, exports them to the tar archive, // imports tar archive to the inStore and checks if all chunks are imported correctly. func ImportExport(t *testing.T, outStore, inStore mock.GlobalStorer, n int) { From 90b6cdaadfb25c2ca117a55d5295a311d6d72447 Mon Sep 17 00:00:00 2001 From: Matthew Halpern Date: Sun, 24 Feb 2019 03:39:23 -0800 Subject: [PATCH 05/21] cmd,swarm: enforce camel case variable names (#19060) --- cmd/swarm/config.go | 104 ++++++++++++++++----------------- cmd/swarm/config_test.go | 4 +- cmd/swarm/flags.go | 38 ++++++------ cmd/swarm/main.go | 6 +- cmd/swarm/upload.go | 4 +- swarm/api/act.go | 2 +- swarm/api/api.go | 8 +-- swarm/fuse/swarmfs.go | 8 +-- swarm/fuse/swarmfs_unix.go | 2 +- swarm/network/stream/peer.go | 16 ++--- swarm/storage/ldbstore_test.go | 8 +-- swarm/swarm.go | 2 +- 12 files changed, 101 insertions(+), 101 deletions(-) diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 98d4dee7b9..0e1e5e39ba 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -59,31 +59,31 @@ var ( //constants for environment variables const ( - SWARM_ENV_CHEQUEBOOK_ADDR = "SWARM_CHEQUEBOOK_ADDR" - SWARM_ENV_ACCOUNT = "SWARM_ACCOUNT" - SWARM_ENV_LISTEN_ADDR = "SWARM_LISTEN_ADDR" - SWARM_ENV_PORT = "SWARM_PORT" - SWARM_ENV_NETWORK_ID = "SWARM_NETWORK_ID" - SWARM_ENV_SWAP_ENABLE = "SWARM_SWAP_ENABLE" - SWARM_ENV_SWAP_API = "SWARM_SWAP_API" - SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE" - SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY" - SWARM_ENV_MAX_STREAM_PEER_SERVERS = "SWARM_ENV_MAX_STREAM_PEER_SERVERS" - SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE" - SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK" - SWARM_ENV_ENS_API = "SWARM_ENS_API" - SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR" - SWARM_ENV_CORS = "SWARM_CORS" - SWARM_ENV_BOOTNODES = "SWARM_BOOTNODES" - SWARM_ENV_PSS_ENABLE = "SWARM_PSS_ENABLE" - SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH" - SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY" - SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY" - SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" - SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" - SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" - SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API" - GETH_ENV_DATADIR = "GETH_DATADIR" + SwarmEnvChequebookAddr = "SWARM_CHEQUEBOOK_ADDR" + SwarmEnvAccount = "SWARM_ACCOUNT" + SwarmEnvListenAddr = "SWARM_LISTEN_ADDR" + SwarmEnvPort = "SWARM_PORT" + SwarmEnvNetworkID = "SWARM_NETWORK_ID" + SwarmEnvSwapEnable = "SWARM_SWAP_ENABLE" + SwarmEnvSwapAPI = "SWARM_SWAP_API" + SwarmEnvSyncDisable = "SWARM_SYNC_DISABLE" + SwarmEnvSyncUpdateDelay = "SWARM_ENV_SYNC_UPDATE_DELAY" + SwarmEnvMaxStreamPeerServers = "SWARM_ENV_MAX_STREAM_PEER_SERVERS" + SwarmEnvLightNodeEnable = "SWARM_LIGHT_NODE_ENABLE" + SwarmEnvDeliverySkipCheck = "SWARM_DELIVERY_SKIP_CHECK" + SwarmEnvENSAPI = "SWARM_ENS_API" + SwarmEnvENSAddr = "SWARM_ENS_ADDR" + SwarmEnvCORS = "SWARM_CORS" + SwarmEnvBootnodes = "SWARM_BOOTNODES" + SwarmEnvPSSEnable = "SWARM_PSS_ENABLE" + SwarmEnvStorePath = "SWARM_STORE_PATH" + SwarmEnvStoreCapacity = "SWARM_STORE_CAPACITY" + SwarmEnvStoreCacheCapacity = "SWARM_STORE_CACHE_CAPACITY" + SwarmEnvBootnodeMode = "SWARM_BOOTNODE_MODE" + SwarmAccessPassword = "SWARM_ACCESS_PASSWORD" + SwarmAutoDefaultPath = "SWARM_AUTO_DEFAULTPATH" + SwarmGlobalstoreAPI = "SWARM_GLOBALSTORE_API" + GethEnvDataDir = "GETH_DATADIR" ) // These settings ensure that TOML keys use the same names as Go struct fields. @@ -227,7 +227,7 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.SwapAPI = ctx.GlobalString(SwarmSwapAPIFlag.Name) if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" { - utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) + utils.Fatalf(SwarmErrSwapSetNoAPI) } if ctx.GlobalIsSet(EnsAPIFlag.Name) { @@ -274,113 +274,113 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con // envVarsOverride overrides the current config with whatver is provided in environment variables // most values are not allowed a zero value (empty string), if not otherwise noted func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { - if keyid := os.Getenv(SWARM_ENV_ACCOUNT); keyid != "" { + if keyid := os.Getenv(SwarmEnvAccount); keyid != "" { currentConfig.BzzAccount = keyid } - if chbookaddr := os.Getenv(SWARM_ENV_CHEQUEBOOK_ADDR); chbookaddr != "" { + if chbookaddr := os.Getenv(SwarmEnvChequebookAddr); chbookaddr != "" { currentConfig.Contract = common.HexToAddress(chbookaddr) } - if networkid := os.Getenv(SWARM_ENV_NETWORK_ID); networkid != "" { + if networkid := os.Getenv(SwarmEnvNetworkID); networkid != "" { id, err := strconv.ParseUint(networkid, 10, 64) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_NETWORK_ID, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvNetworkID, err) } if id != 0 { currentConfig.NetworkID = id } } - if datadir := os.Getenv(GETH_ENV_DATADIR); datadir != "" { + if datadir := os.Getenv(GethEnvDataDir); datadir != "" { currentConfig.Path = expandPath(datadir) } - bzzport := os.Getenv(SWARM_ENV_PORT) + bzzport := os.Getenv(SwarmEnvPort) if len(bzzport) > 0 { currentConfig.Port = bzzport } - if bzzaddr := os.Getenv(SWARM_ENV_LISTEN_ADDR); bzzaddr != "" { + if bzzaddr := os.Getenv(SwarmEnvListenAddr); bzzaddr != "" { currentConfig.ListenAddr = bzzaddr } - if swapenable := os.Getenv(SWARM_ENV_SWAP_ENABLE); swapenable != "" { + if swapenable := os.Getenv(SwarmEnvSwapEnable); swapenable != "" { swap, err := strconv.ParseBool(swapenable) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SWAP_ENABLE, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSwapEnable, err) } currentConfig.SwapEnabled = swap } - if syncdisable := os.Getenv(SWARM_ENV_SYNC_DISABLE); syncdisable != "" { + if syncdisable := os.Getenv(SwarmEnvSyncDisable); syncdisable != "" { sync, err := strconv.ParseBool(syncdisable) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_DISABLE, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncDisable, err) } currentConfig.SyncEnabled = !sync } - if v := os.Getenv(SWARM_ENV_DELIVERY_SKIP_CHECK); v != "" { + if v := os.Getenv(SwarmEnvDeliverySkipCheck); v != "" { skipCheck, err := strconv.ParseBool(v) if err != nil { currentConfig.DeliverySkipCheck = skipCheck } } - if v := os.Getenv(SWARM_ENV_SYNC_UPDATE_DELAY); v != "" { + if v := os.Getenv(SwarmEnvSyncUpdateDelay); v != "" { d, err := time.ParseDuration(v) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_SYNC_UPDATE_DELAY, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvSyncUpdateDelay, err) } currentConfig.SyncUpdateDelay = d } - if max := os.Getenv(SWARM_ENV_MAX_STREAM_PEER_SERVERS); max != "" { + if max := os.Getenv(SwarmEnvMaxStreamPeerServers); max != "" { m, err := strconv.Atoi(max) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_MAX_STREAM_PEER_SERVERS, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvMaxStreamPeerServers, err) } currentConfig.MaxStreamPeerServers = m } - if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" { + if lne := os.Getenv(SwarmEnvLightNodeEnable); lne != "" { lightnode, err := strconv.ParseBool(lne) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_LIGHT_NODE_ENABLE, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvLightNodeEnable, err) } currentConfig.LightNodeEnabled = lightnode } - if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" { + if swapapi := os.Getenv(SwarmEnvSwapAPI); swapapi != "" { currentConfig.SwapAPI = swapapi } if currentConfig.SwapEnabled && currentConfig.SwapAPI == "" { - utils.Fatalf(SWARM_ERR_SWAP_SET_NO_API) + utils.Fatalf(SwarmErrSwapSetNoAPI) } - if ensapi := os.Getenv(SWARM_ENV_ENS_API); ensapi != "" { + if ensapi := os.Getenv(SwarmEnvENSAPI); ensapi != "" { currentConfig.EnsAPIs = strings.Split(ensapi, ",") } - if ensaddr := os.Getenv(SWARM_ENV_ENS_ADDR); ensaddr != "" { + if ensaddr := os.Getenv(SwarmEnvENSAddr); ensaddr != "" { currentConfig.EnsRoot = common.HexToAddress(ensaddr) } - if cors := os.Getenv(SWARM_ENV_CORS); cors != "" { + if cors := os.Getenv(SwarmEnvCORS); cors != "" { currentConfig.Cors = cors } - if bm := os.Getenv(SWARM_ENV_BOOTNODE_MODE); bm != "" { + if bm := os.Getenv(SwarmEnvBootnodeMode); bm != "" { bootnodeMode, err := strconv.ParseBool(bm) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_ENV_BOOTNODE_MODE, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmEnvBootnodeMode, err) } currentConfig.BootnodeMode = bootnodeMode } - if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" { + if api := os.Getenv(SwarmGlobalstoreAPI); api != "" { currentConfig.GlobalStoreAPI = api } diff --git a/cmd/swarm/config_test.go b/cmd/swarm/config_test.go index 18be316e5f..869edd0f70 100644 --- a/cmd/swarm/config_test.go +++ b/cmd/swarm/config_test.go @@ -52,7 +52,7 @@ func TestConfigFailsSwapEnabledNoSwapApi(t *testing.T) { } swarm := runSwarm(t, flags...) - swarm.Expect("Fatal: " + SWARM_ERR_SWAP_SET_NO_API + "\n") + swarm.Expect("Fatal: " + SwarmErrSwapSetNoAPI + "\n") swarm.ExpectExit() } @@ -63,7 +63,7 @@ func TestConfigFailsNoBzzAccount(t *testing.T) { } swarm := runSwarm(t, flags...) - swarm.Expect("Fatal: " + SWARM_ERR_NO_BZZACCOUNT + "\n") + swarm.Expect("Fatal: " + SwarmErrNoBZZAccount + "\n") swarm.ExpectExit() } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index b092a77476..39a273d870 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -23,68 +23,68 @@ var ( ChequebookAddrFlag = cli.StringFlag{ Name: "chequebook", Usage: "chequebook contract address", - EnvVar: SWARM_ENV_CHEQUEBOOK_ADDR, + EnvVar: SwarmEnvChequebookAddr, } SwarmAccountFlag = cli.StringFlag{ Name: "bzzaccount", Usage: "Swarm account key file", - EnvVar: SWARM_ENV_ACCOUNT, + EnvVar: SwarmEnvAccount, } SwarmListenAddrFlag = cli.StringFlag{ Name: "httpaddr", Usage: "Swarm HTTP API listening interface", - EnvVar: SWARM_ENV_LISTEN_ADDR, + EnvVar: SwarmEnvListenAddr, } SwarmPortFlag = cli.StringFlag{ Name: "bzzport", Usage: "Swarm local http api port", - EnvVar: SWARM_ENV_PORT, + EnvVar: SwarmEnvPort, } SwarmNetworkIdFlag = cli.IntFlag{ Name: "bzznetworkid", Usage: "Network identifier (integer, default 3=swarm testnet)", - EnvVar: SWARM_ENV_NETWORK_ID, + EnvVar: SwarmEnvNetworkID, } SwarmSwapEnabledFlag = cli.BoolFlag{ Name: "swap", Usage: "Swarm SWAP enabled (default false)", - EnvVar: SWARM_ENV_SWAP_ENABLE, + EnvVar: SwarmEnvSwapEnable, } SwarmSwapAPIFlag = cli.StringFlag{ Name: "swap-api", Usage: "URL of the Ethereum API provider to use to settle SWAP payments", - EnvVar: SWARM_ENV_SWAP_API, + EnvVar: SwarmEnvSwapAPI, } SwarmSyncDisabledFlag = cli.BoolTFlag{ Name: "nosync", Usage: "Disable swarm syncing", - EnvVar: SWARM_ENV_SYNC_DISABLE, + EnvVar: SwarmEnvSyncDisable, } SwarmSyncUpdateDelay = cli.DurationFlag{ Name: "sync-update-delay", Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)", - EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY, + EnvVar: SwarmEnvSyncUpdateDelay, } SwarmMaxStreamPeerServersFlag = cli.IntFlag{ Name: "max-stream-peer-servers", Usage: "Limit of Stream peer servers, 0 denotes unlimited", - EnvVar: SWARM_ENV_MAX_STREAM_PEER_SERVERS, + EnvVar: SwarmEnvMaxStreamPeerServers, Value: 10000, // A very large default value is possible as stream servers have very small memory footprint } SwarmLightNodeEnabled = cli.BoolFlag{ Name: "lightnode", Usage: "Enable Swarm LightNode (default false)", - EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE, + EnvVar: SwarmEnvLightNodeEnable, } SwarmDeliverySkipCheckFlag = cli.BoolFlag{ Name: "delivery-skip-check", Usage: "Skip chunk delivery check (default false)", - EnvVar: SWARM_ENV_DELIVERY_SKIP_CHECK, + EnvVar: SwarmEnvDeliverySkipCheck, } EnsAPIFlag = cli.StringSliceFlag{ Name: "ens-api", Usage: "ENS API endpoint for a TLD and with contract address, can be repeated, format [tld:][contract-addr@]url", - EnvVar: SWARM_ENV_ENS_API, + EnvVar: SwarmEnvENSAPI, } SwarmApiFlag = cli.StringFlag{ Name: "bzzapi", @@ -126,7 +126,7 @@ var ( SwarmAccessPasswordFlag = cli.StringFlag{ Name: "password", Usage: "Password", - EnvVar: SWARM_ACCESS_PASSWORD, + EnvVar: SwarmAccessPassword, } SwarmDryRunFlag = cli.BoolFlag{ Name: "dry-run", @@ -135,22 +135,22 @@ var ( CorsStringFlag = cli.StringFlag{ Name: "corsdomain", Usage: "Domain on which to send Access-Control-Allow-Origin header (multiple domains can be supplied separated by a ',')", - EnvVar: SWARM_ENV_CORS, + EnvVar: SwarmEnvCORS, } SwarmStorePath = cli.StringFlag{ Name: "store.path", Usage: "Path to leveldb chunk DB (default <$GETH_ENV_DIR>/swarm/bzz-<$BZZ_KEY>/chunks)", - EnvVar: SWARM_ENV_STORE_PATH, + EnvVar: SwarmEnvStorePath, } SwarmStoreCapacity = cli.Uint64Flag{ Name: "store.size", Usage: "Number of chunks (5M is roughly 20-25GB) (default 5000000)", - EnvVar: SWARM_ENV_STORE_CAPACITY, + EnvVar: SwarmEnvStoreCapacity, } SwarmStoreCacheCapacity = cli.UintFlag{ Name: "store.cache.size", Usage: "Number of recent chunks cached in memory (default 5000)", - EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY, + EnvVar: SwarmEnvStoreCacheCapacity, } SwarmCompressedFlag = cli.BoolFlag{ Name: "compressed", @@ -179,6 +179,6 @@ var ( SwarmGlobalStoreAPIFlag = cli.StringFlag{ Name: "globalstore-api", Usage: "URL of the Global Store API provider (only for testing)", - EnvVar: SWARM_GLOBALSTORE_API, + EnvVar: SwarmGlobalstoreAPI, } ) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index af00503ea3..4d63255d7a 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -76,8 +76,8 @@ var gitCommit string //declare a few constant error messages, useful for later error check comparisons in test var ( - SWARM_ERR_NO_BZZACCOUNT = "bzzaccount option is required but not set; check your config file, command line or environment variables" - SWARM_ERR_SWAP_SET_NO_API = "SWAP is enabled but --swap-api is not set" + SwarmErrNoBZZAccount = "bzzaccount option is required but not set; check your config file, command line or environment variables" + SwarmErrSwapSetNoAPI = "SWAP is enabled but --swap-api is not set" ) // this help command gets added to any subcommand that does not define it explicitly @@ -351,7 +351,7 @@ func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) { func getAccount(bzzaccount string, ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { //an account is mandatory if bzzaccount == "" { - utils.Fatalf(SWARM_ERR_NO_BZZACCOUNT) + utils.Fatalf(SwarmErrNoBZZAccount) } // Try to load the arg as a hex key file. if key, err := crypto.LoadECDSA(bzzaccount); err == nil { diff --git a/cmd/swarm/upload.go b/cmd/swarm/upload.go index 992f2d6e97..ab07908351 100644 --- a/cmd/swarm/upload.go +++ b/cmd/swarm/upload.go @@ -60,10 +60,10 @@ func upload(ctx *cli.Context) { autoDefaultPath = false file string ) - if autoDefaultPathString := os.Getenv(SWARM_AUTO_DEFAULTPATH); autoDefaultPathString != "" { + if autoDefaultPathString := os.Getenv(SwarmAutoDefaultPath); autoDefaultPathString != "" { b, err := strconv.ParseBool(autoDefaultPathString) if err != nil { - utils.Fatalf("invalid environment variable %s: %v", SWARM_AUTO_DEFAULTPATH, err) + utils.Fatalf("invalid environment variable %s: %v", SwarmAutoDefaultPath, err) } autoDefaultPath = b } diff --git a/swarm/api/act.go b/swarm/api/act.go index 9566720b03..a79f1944b9 100644 --- a/swarm/api/act.go +++ b/swarm/api/act.go @@ -33,7 +33,7 @@ var ( } ) -const EMPTY_CREDENTIALS = "" +const EmptyCredentials = "" type AccessEntry struct { Type AccessType diff --git a/swarm/api/api.go b/swarm/api/api.go index c6ca1b5774..86c1119232 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -431,7 +431,7 @@ func (a *API) Delete(ctx context.Context, addr string, path string) (storage.Add apiDeleteFail.Inc(1) return nil, err } - key, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) + key, err := a.ResolveURI(ctx, uri, EmptyCredentials) if err != nil { return nil, err @@ -643,7 +643,7 @@ func (a *API) AddFile(ctx context.Context, mhash, path, fname string, content [] apiAddFileFail.Inc(1) return nil, "", err } - mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) + mkey, err := a.ResolveURI(ctx, uri, EmptyCredentials) if err != nil { apiAddFileFail.Inc(1) return nil, "", err @@ -760,7 +760,7 @@ func (a *API) RemoveFile(ctx context.Context, mhash string, path string, fname s apiRmFileFail.Inc(1) return "", err } - mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) + mkey, err := a.ResolveURI(ctx, uri, EmptyCredentials) if err != nil { apiRmFileFail.Inc(1) return "", err @@ -827,7 +827,7 @@ func (a *API) AppendFile(ctx context.Context, mhash, path, fname string, existin apiAppendFileFail.Inc(1) return nil, "", err } - mkey, err := a.ResolveURI(ctx, uri, EMPTY_CREDENTIALS) + mkey, err := a.ResolveURI(ctx, uri, EmptyCredentials) if err != nil { apiAppendFileFail.Inc(1) return nil, "", err diff --git a/swarm/fuse/swarmfs.go b/swarm/fuse/swarmfs.go index c7aa983b7d..db6aefb54c 100644 --- a/swarm/fuse/swarmfs.go +++ b/swarm/fuse/swarmfs.go @@ -24,10 +24,10 @@ import ( ) const ( - Swarmfs_Version = "0.1" - mountTimeout = time.Second * 5 - unmountTimeout = time.Second * 10 - maxFuseMounts = 5 + SwarmFSVersion = "0.1" + mountTimeout = time.Second * 5 + unmountTimeout = time.Second * 10 + maxFUSEMounts = 5 ) var ( diff --git a/swarm/fuse/swarmfs_unix.go b/swarm/fuse/swarmfs_unix.go index 9ff55cc324..54b879a4da 100644 --- a/swarm/fuse/swarmfs_unix.go +++ b/swarm/fuse/swarmfs_unix.go @@ -96,7 +96,7 @@ func (swarmfs *SwarmFS) Mount(mhash, mountpoint string) (*MountInfo, error) { noOfActiveMounts := len(swarmfs.activeMounts) log.Debug("swarmfs mount", "# active mounts", noOfActiveMounts) - if noOfActiveMounts >= maxFuseMounts { + if noOfActiveMounts >= maxFUSEMounts { return nil, errMaxMountCount } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index c59799e08a..1d3868a66b 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -101,20 +101,20 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { for { select { case <-ticker.C: - var len_maxi int - var cap_maxi int + var lenMaxi int + var capMaxi int for k := range pq.Queues { - if len_maxi < len(pq.Queues[k]) { - len_maxi = len(pq.Queues[k]) + if lenMaxi < len(pq.Queues[k]) { + lenMaxi = len(pq.Queues[k]) } - if cap_maxi < cap(pq.Queues[k]) { - cap_maxi = cap(pq.Queues[k]) + if capMaxi < cap(pq.Queues[k]) { + capMaxi = cap(pq.Queues[k]) } } - metrics.GetOrRegisterGauge(fmt.Sprintf("pq_len_%s", p.ID().TerminalString()), nil).Update(int64(len_maxi)) - metrics.GetOrRegisterGauge(fmt.Sprintf("pq_cap_%s", p.ID().TerminalString()), nil).Update(int64(cap_maxi)) + metrics.GetOrRegisterGauge(fmt.Sprintf("pq_len_%s", p.ID().TerminalString()), nil).Update(int64(lenMaxi)) + metrics.GetOrRegisterGauge(fmt.Sprintf("pq_cap_%s", p.ID().TerminalString()), nil).Update(int64(capMaxi)) case <-p.quit: return } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 9e7aba545d..65b72acec8 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -193,7 +193,7 @@ func testIterator(t *testing.T, mock bool) { var i int var poc uint chunkkeys := NewAddressCollection(chunkcount) - chunkkeys_results := NewAddressCollection(chunkcount) + chunkkeysResults := NewAddressCollection(chunkcount) db, cleanup, err := newTestDbStore(mock, false) defer cleanup() @@ -218,7 +218,7 @@ func testIterator(t *testing.T, mock bool) { for poc = 0; poc <= 255; poc++ { err := db.SyncIterator(0, uint64(chunkkeys.Len()), uint8(poc), func(k Address, n uint64) bool { log.Trace(fmt.Sprintf("Got key %v number %d poc %d", k, n, uint8(poc))) - chunkkeys_results[n] = k + chunkkeysResults[n] = k i++ return true }) @@ -228,8 +228,8 @@ func testIterator(t *testing.T, mock bool) { } for i = 0; i < chunkcount; i++ { - if !bytes.Equal(chunkkeys[i], chunkkeys_results[i]) { - t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeys_results[i]) + if !bytes.Equal(chunkkeys[i], chunkkeysResults[i]) { + t.Fatalf("Chunk put #%d key '%v' does not match iterator's key '%v'", i, chunkkeys[i], chunkkeysResults[i]) } } diff --git a/swarm/swarm.go b/swarm/swarm.go index 651ad97c78..b4b08c5c54 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -508,7 +508,7 @@ func (s *Swarm) APIs() []rpc.API { }, { Namespace: "swarmfs", - Version: fuse.Swarmfs_Version, + Version: fuse.SwarmFSVersion, Service: s.sfs, Public: false, }, From 4e87b444ca3fc6b808c5c969b571af721c6d6252 Mon Sep 17 00:00:00 2001 From: Matthew Halpern Date: Sun, 24 Feb 2019 03:41:22 -0800 Subject: [PATCH 06/21] swarm/pss: remove unused function (#19100) --- swarm/pss/pss_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 675b4cfcd6..ea7a591b1e 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -38,8 +38,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/metrics/influxdb" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -1369,8 +1367,6 @@ func TestNetwork(t *testing.T) { // nodes/msgs/addrbytes/adaptertype // if adaptertype is exec uses execadapter, simadapter otherwise func TestNetwork2000(t *testing.T) { - //enableMetrics() - if !*longrunning { t.Skip("run with --longrunning flag to run extensive network tests") } @@ -1381,8 +1377,6 @@ func TestNetwork2000(t *testing.T) { } func TestNetwork5000(t *testing.T) { - //enableMetrics() - if !*longrunning { t.Skip("run with --longrunning flag to run extensive network tests") } @@ -1393,8 +1387,6 @@ func TestNetwork5000(t *testing.T) { } func TestNetwork10000(t *testing.T) { - //enableMetrics() - if !*longrunning { t.Skip("run with --longrunning flag to run extensive network tests") } @@ -2098,11 +2090,3 @@ func (apitest *APITest) SetSymKeys(pubkeyid string, recvsymkey []byte, sendsymke func (apitest *APITest) Clean() (int, error) { return apitest.Pss.cleanKeys(), nil } - -// enableMetrics is starting InfluxDB reporter so that we collect stats when running tests locally -func enableMetrics() { - metrics.Enabled = true - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 1*time.Second, "http://localhost:8086", "metrics", "admin", "admin", "swarm.", map[string]string{ - "host": "test", - }) -} From 81babe15090e4ac8a8eea1cb07999f9700738786 Mon Sep 17 00:00:00 2001 From: Matthew Halpern Date: Sun, 24 Feb 2019 23:58:18 -0800 Subject: [PATCH 07/21] swarm/*: remove redundant type specifiers (#19089) --- swarm/network/stream/stream.go | 2 +- swarm/pss/pss.go | 2 +- swarm/storage/feed/lookup/lookup_test.go | 2 +- swarm/storage/ldbstore_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 3bc4504558..c7c489152a 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -75,7 +75,7 @@ const ( // subscriptionFunc is used to determine what to do in order to perform subscriptions // usually we would start to really subscribe to nodes, but for tests other functionality may be needed // (see TestRequestPeerSubscriptions in streamer_test.go) -var subscriptionFunc func(r *Registry, p *network.Peer, bin uint8, subs map[enode.ID]map[Stream]struct{}) bool = doRequestSubscription +var subscriptionFunc = doRequestSubscription // Registry registry for outgoing and incoming streamer constructors type Registry struct { diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index 0b8cc148c5..0a8d757d8a 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -676,7 +676,7 @@ func (p *Pss) send(to []byte, topic Topic, msg []byte, asymmetric bool, key []by // sendFunc is a helper function that tries to send a message and returns true on success. // It is set here for usage in production, and optionally overridden in tests. -var sendFunc func(p *Pss, sp *network.Peer, msg *PssMsg) bool = sendMsg +var sendFunc = sendMsg // tries to send a message, returns true if successful func sendMsg(p *Pss, sp *network.Peer, msg *PssMsg) bool { diff --git a/swarm/storage/feed/lookup/lookup_test.go b/swarm/storage/feed/lookup/lookup_test.go index d71e81e95f..4652feacfc 100644 --- a/swarm/storage/feed/lookup/lookup_test.go +++ b/swarm/storage/feed/lookup/lookup_test.go @@ -346,7 +346,7 @@ type testG struct { } // test cases -var testGetNextLevelCases []testG = []testG{{e: lookup.Epoch{Time: 989875233, Level: 12}, n: 989875233, x: 11}, {e: lookup.Epoch{Time: 995807650, Level: 18}, n: 995598156, x: 19}, {e: lookup.Epoch{Time: 969167082, Level: 0}, n: 968990357, x: 18}, {e: lookup.Epoch{Time: 993087628, Level: 14}, n: 992987044, x: 20}, {e: lookup.Epoch{Time: 963364631, Level: 20}, n: 963364630, x: 19}, {e: lookup.Epoch{Time: 963497510, Level: 16}, n: 963370732, x: 18}, {e: lookup.Epoch{Time: 955421349, Level: 22}, n: 955421348, x: 21}, {e: lookup.Epoch{Time: 968220379, Level: 15}, n: 968220378, x: 14}, {e: lookup.Epoch{Time: 939129014, Level: 6}, n: 939128771, x: 11}, {e: lookup.Epoch{Time: 907847903, Level: 6}, n: 907791833, x: 18}, {e: lookup.Epoch{Time: 910835564, Level: 15}, n: 910835564, x: 14}, {e: lookup.Epoch{Time: 913578333, Level: 22}, n: 881808431, x: 25}, {e: lookup.Epoch{Time: 895818460, Level: 3}, n: 895818132, x: 9}, {e: lookup.Epoch{Time: 903843025, Level: 24}, n: 895609561, x: 23}, {e: lookup.Epoch{Time: 877889433, Level: 13}, n: 877877093, x: 15}, {e: lookup.Epoch{Time: 901450396, Level: 10}, n: 901450058, x: 9}, {e: lookup.Epoch{Time: 925179910, Level: 3}, n: 925168393, x: 16}, {e: lookup.Epoch{Time: 913485477, Level: 21}, n: 913485476, x: 20}, {e: lookup.Epoch{Time: 924462991, Level: 18}, n: 924462990, x: 17}, {e: lookup.Epoch{Time: 941175128, Level: 13}, n: 941175127, x: 12}, {e: lookup.Epoch{Time: 920126583, Level: 3}, n: 920100782, x: 19}, {e: lookup.Epoch{Time: 932403200, Level: 9}, n: 932279891, x: 17}, {e: lookup.Epoch{Time: 948284931, Level: 2}, n: 948284921, x: 9}, {e: lookup.Epoch{Time: 953540997, Level: 7}, n: 950547986, x: 22}, {e: lookup.Epoch{Time: 926639837, Level: 18}, n: 918608882, x: 24}, {e: lookup.Epoch{Time: 954637598, Level: 1}, n: 954578761, x: 17}, {e: lookup.Epoch{Time: 943482981, Level: 10}, n: 942924151, x: 19}, {e: lookup.Epoch{Time: 963580771, Level: 7}, n: 963580771, x: 6}, {e: lookup.Epoch{Time: 993744930, Level: 7}, n: 993690858, x: 16}, {e: lookup.Epoch{Time: 1018890213, Level: 12}, n: 1018890212, x: 11}, {e: lookup.Epoch{Time: 1030309411, Level: 2}, n: 1030309227, x: 9}, {e: lookup.Epoch{Time: 1063204997, Level: 20}, n: 1063204996, x: 19}, {e: lookup.Epoch{Time: 1094340832, Level: 6}, n: 1094340633, x: 7}, {e: lookup.Epoch{Time: 1077880597, Level: 10}, n: 1075914292, x: 20}, {e: lookup.Epoch{Time: 1051114957, Level: 18}, n: 1051114957, x: 17}, {e: lookup.Epoch{Time: 1045649701, Level: 22}, n: 1045649700, x: 21}, {e: lookup.Epoch{Time: 1066198885, Level: 14}, n: 1066198884, x: 13}, {e: lookup.Epoch{Time: 1053231952, Level: 1}, n: 1053210845, x: 16}, {e: lookup.Epoch{Time: 1068763404, Level: 14}, n: 1068675428, x: 18}, {e: lookup.Epoch{Time: 1039042173, Level: 15}, n: 1038973110, x: 17}, {e: lookup.Epoch{Time: 1050747636, Level: 6}, n: 1050747364, x: 9}, {e: lookup.Epoch{Time: 1030034434, Level: 23}, n: 1030034433, x: 22}, {e: lookup.Epoch{Time: 1003783425, Level: 18}, n: 1003783424, x: 17}, {e: lookup.Epoch{Time: 988163976, Level: 15}, n: 988084064, x: 17}, {e: lookup.Epoch{Time: 1007222377, Level: 15}, n: 1007222377, x: 14}, {e: lookup.Epoch{Time: 1001211375, Level: 13}, n: 1001208178, x: 14}, {e: lookup.Epoch{Time: 997623199, Level: 8}, n: 997623198, x: 7}, {e: lookup.Epoch{Time: 1026283830, Level: 10}, n: 1006681704, x: 24}, {e: lookup.Epoch{Time: 1019421907, Level: 20}, n: 1019421906, x: 19}, {e: lookup.Epoch{Time: 1043154306, Level: 16}, n: 1043108343, x: 16}, {e: lookup.Epoch{Time: 1075643767, Level: 17}, n: 1075325898, x: 18}, {e: lookup.Epoch{Time: 1043726309, Level: 20}, n: 1043726308, x: 19}, {e: lookup.Epoch{Time: 1056415324, Level: 17}, n: 1056415324, x: 16}, {e: lookup.Epoch{Time: 1088650219, Level: 13}, n: 1088650218, x: 12}, {e: lookup.Epoch{Time: 1088551662, Level: 7}, n: 1088543355, x: 13}, {e: lookup.Epoch{Time: 1069667265, Level: 6}, n: 1069667075, x: 7}, {e: lookup.Epoch{Time: 1079145970, Level: 18}, n: 1079145969, x: 17}, {e: lookup.Epoch{Time: 1083338876, Level: 7}, n: 1083338875, x: 6}, {e: lookup.Epoch{Time: 1051581086, Level: 4}, n: 1051568869, x: 14}, {e: lookup.Epoch{Time: 1028430882, Level: 4}, n: 1028430864, x: 5}, {e: lookup.Epoch{Time: 1057356462, Level: 1}, n: 1057356417, x: 5}, {e: lookup.Epoch{Time: 1033104266, Level: 0}, n: 1033097479, x: 13}, {e: lookup.Epoch{Time: 1031391367, Level: 11}, n: 1031387304, x: 14}, {e: lookup.Epoch{Time: 1049781164, Level: 15}, n: 1049781163, x: 14}, {e: lookup.Epoch{Time: 1027271628, Level: 12}, n: 1027271627, x: 11}, {e: lookup.Epoch{Time: 1057270560, Level: 23}, n: 1057270560, x: 22}, {e: lookup.Epoch{Time: 1047501317, Level: 15}, n: 1047501317, x: 14}, {e: lookup.Epoch{Time: 1058349035, Level: 11}, n: 1045175573, x: 24}, {e: lookup.Epoch{Time: 1057396147, Level: 20}, n: 1057396147, x: 19}, {e: lookup.Epoch{Time: 1048906375, Level: 18}, n: 1039616919, x: 25}, {e: lookup.Epoch{Time: 1074294831, Level: 20}, n: 1074294831, x: 19}, {e: lookup.Epoch{Time: 1088946052, Level: 1}, n: 1088917364, x: 14}, {e: lookup.Epoch{Time: 1112337595, Level: 17}, n: 1111008110, x: 22}, {e: lookup.Epoch{Time: 1099990284, Level: 5}, n: 1099968370, x: 15}, {e: lookup.Epoch{Time: 1087036441, Level: 16}, n: 1053967855, x: 25}, {e: lookup.Epoch{Time: 1069225185, Level: 8}, n: 1069224660, x: 10}, {e: lookup.Epoch{Time: 1057505479, Level: 9}, n: 1057505170, x: 14}, {e: lookup.Epoch{Time: 1072381377, Level: 12}, n: 1065950959, x: 22}, {e: lookup.Epoch{Time: 1093887139, Level: 8}, n: 1093863305, x: 14}, {e: lookup.Epoch{Time: 1082366510, Level: 24}, n: 1082366510, x: 23}, {e: lookup.Epoch{Time: 1103231132, Level: 14}, n: 1102292201, x: 22}, {e: lookup.Epoch{Time: 1094502355, Level: 3}, n: 1094324652, x: 18}, {e: lookup.Epoch{Time: 1068488344, Level: 12}, n: 1067577330, x: 19}, {e: lookup.Epoch{Time: 1050278233, Level: 12}, n: 1050278232, x: 11}, {e: lookup.Epoch{Time: 1047660768, Level: 5}, n: 1047652137, x: 17}, {e: lookup.Epoch{Time: 1060116167, Level: 11}, n: 1060114091, x: 12}, {e: lookup.Epoch{Time: 1068149392, Level: 21}, n: 1052074801, x: 24}, {e: lookup.Epoch{Time: 1081934120, Level: 6}, n: 1081933847, x: 8}, {e: lookup.Epoch{Time: 1107943693, Level: 16}, n: 1107096139, x: 25}, {e: lookup.Epoch{Time: 1131571649, Level: 9}, n: 1131570428, x: 11}, {e: lookup.Epoch{Time: 1123139367, Level: 0}, n: 1122912198, x: 20}, {e: lookup.Epoch{Time: 1121144423, Level: 6}, n: 1120568289, x: 20}, {e: lookup.Epoch{Time: 1089932411, Level: 17}, n: 1089932410, x: 16}, {e: lookup.Epoch{Time: 1104899012, Level: 22}, n: 1098978789, x: 22}, {e: lookup.Epoch{Time: 1094588059, Level: 21}, n: 1094588059, x: 20}, {e: lookup.Epoch{Time: 1114987438, Level: 24}, n: 1114987437, x: 23}, {e: lookup.Epoch{Time: 1084186305, Level: 7}, n: 1084186241, x: 6}, {e: lookup.Epoch{Time: 1058827111, Level: 8}, n: 1058826504, x: 9}, {e: lookup.Epoch{Time: 1090679810, Level: 12}, n: 1090616539, x: 17}, {e: lookup.Epoch{Time: 1084299475, Level: 23}, n: 1084299475, x: 22}} +var testGetNextLevelCases = []testG{{e: lookup.Epoch{Time: 989875233, Level: 12}, n: 989875233, x: 11}, {e: lookup.Epoch{Time: 995807650, Level: 18}, n: 995598156, x: 19}, {e: lookup.Epoch{Time: 969167082, Level: 0}, n: 968990357, x: 18}, {e: lookup.Epoch{Time: 993087628, Level: 14}, n: 992987044, x: 20}, {e: lookup.Epoch{Time: 963364631, Level: 20}, n: 963364630, x: 19}, {e: lookup.Epoch{Time: 963497510, Level: 16}, n: 963370732, x: 18}, {e: lookup.Epoch{Time: 955421349, Level: 22}, n: 955421348, x: 21}, {e: lookup.Epoch{Time: 968220379, Level: 15}, n: 968220378, x: 14}, {e: lookup.Epoch{Time: 939129014, Level: 6}, n: 939128771, x: 11}, {e: lookup.Epoch{Time: 907847903, Level: 6}, n: 907791833, x: 18}, {e: lookup.Epoch{Time: 910835564, Level: 15}, n: 910835564, x: 14}, {e: lookup.Epoch{Time: 913578333, Level: 22}, n: 881808431, x: 25}, {e: lookup.Epoch{Time: 895818460, Level: 3}, n: 895818132, x: 9}, {e: lookup.Epoch{Time: 903843025, Level: 24}, n: 895609561, x: 23}, {e: lookup.Epoch{Time: 877889433, Level: 13}, n: 877877093, x: 15}, {e: lookup.Epoch{Time: 901450396, Level: 10}, n: 901450058, x: 9}, {e: lookup.Epoch{Time: 925179910, Level: 3}, n: 925168393, x: 16}, {e: lookup.Epoch{Time: 913485477, Level: 21}, n: 913485476, x: 20}, {e: lookup.Epoch{Time: 924462991, Level: 18}, n: 924462990, x: 17}, {e: lookup.Epoch{Time: 941175128, Level: 13}, n: 941175127, x: 12}, {e: lookup.Epoch{Time: 920126583, Level: 3}, n: 920100782, x: 19}, {e: lookup.Epoch{Time: 932403200, Level: 9}, n: 932279891, x: 17}, {e: lookup.Epoch{Time: 948284931, Level: 2}, n: 948284921, x: 9}, {e: lookup.Epoch{Time: 953540997, Level: 7}, n: 950547986, x: 22}, {e: lookup.Epoch{Time: 926639837, Level: 18}, n: 918608882, x: 24}, {e: lookup.Epoch{Time: 954637598, Level: 1}, n: 954578761, x: 17}, {e: lookup.Epoch{Time: 943482981, Level: 10}, n: 942924151, x: 19}, {e: lookup.Epoch{Time: 963580771, Level: 7}, n: 963580771, x: 6}, {e: lookup.Epoch{Time: 993744930, Level: 7}, n: 993690858, x: 16}, {e: lookup.Epoch{Time: 1018890213, Level: 12}, n: 1018890212, x: 11}, {e: lookup.Epoch{Time: 1030309411, Level: 2}, n: 1030309227, x: 9}, {e: lookup.Epoch{Time: 1063204997, Level: 20}, n: 1063204996, x: 19}, {e: lookup.Epoch{Time: 1094340832, Level: 6}, n: 1094340633, x: 7}, {e: lookup.Epoch{Time: 1077880597, Level: 10}, n: 1075914292, x: 20}, {e: lookup.Epoch{Time: 1051114957, Level: 18}, n: 1051114957, x: 17}, {e: lookup.Epoch{Time: 1045649701, Level: 22}, n: 1045649700, x: 21}, {e: lookup.Epoch{Time: 1066198885, Level: 14}, n: 1066198884, x: 13}, {e: lookup.Epoch{Time: 1053231952, Level: 1}, n: 1053210845, x: 16}, {e: lookup.Epoch{Time: 1068763404, Level: 14}, n: 1068675428, x: 18}, {e: lookup.Epoch{Time: 1039042173, Level: 15}, n: 1038973110, x: 17}, {e: lookup.Epoch{Time: 1050747636, Level: 6}, n: 1050747364, x: 9}, {e: lookup.Epoch{Time: 1030034434, Level: 23}, n: 1030034433, x: 22}, {e: lookup.Epoch{Time: 1003783425, Level: 18}, n: 1003783424, x: 17}, {e: lookup.Epoch{Time: 988163976, Level: 15}, n: 988084064, x: 17}, {e: lookup.Epoch{Time: 1007222377, Level: 15}, n: 1007222377, x: 14}, {e: lookup.Epoch{Time: 1001211375, Level: 13}, n: 1001208178, x: 14}, {e: lookup.Epoch{Time: 997623199, Level: 8}, n: 997623198, x: 7}, {e: lookup.Epoch{Time: 1026283830, Level: 10}, n: 1006681704, x: 24}, {e: lookup.Epoch{Time: 1019421907, Level: 20}, n: 1019421906, x: 19}, {e: lookup.Epoch{Time: 1043154306, Level: 16}, n: 1043108343, x: 16}, {e: lookup.Epoch{Time: 1075643767, Level: 17}, n: 1075325898, x: 18}, {e: lookup.Epoch{Time: 1043726309, Level: 20}, n: 1043726308, x: 19}, {e: lookup.Epoch{Time: 1056415324, Level: 17}, n: 1056415324, x: 16}, {e: lookup.Epoch{Time: 1088650219, Level: 13}, n: 1088650218, x: 12}, {e: lookup.Epoch{Time: 1088551662, Level: 7}, n: 1088543355, x: 13}, {e: lookup.Epoch{Time: 1069667265, Level: 6}, n: 1069667075, x: 7}, {e: lookup.Epoch{Time: 1079145970, Level: 18}, n: 1079145969, x: 17}, {e: lookup.Epoch{Time: 1083338876, Level: 7}, n: 1083338875, x: 6}, {e: lookup.Epoch{Time: 1051581086, Level: 4}, n: 1051568869, x: 14}, {e: lookup.Epoch{Time: 1028430882, Level: 4}, n: 1028430864, x: 5}, {e: lookup.Epoch{Time: 1057356462, Level: 1}, n: 1057356417, x: 5}, {e: lookup.Epoch{Time: 1033104266, Level: 0}, n: 1033097479, x: 13}, {e: lookup.Epoch{Time: 1031391367, Level: 11}, n: 1031387304, x: 14}, {e: lookup.Epoch{Time: 1049781164, Level: 15}, n: 1049781163, x: 14}, {e: lookup.Epoch{Time: 1027271628, Level: 12}, n: 1027271627, x: 11}, {e: lookup.Epoch{Time: 1057270560, Level: 23}, n: 1057270560, x: 22}, {e: lookup.Epoch{Time: 1047501317, Level: 15}, n: 1047501317, x: 14}, {e: lookup.Epoch{Time: 1058349035, Level: 11}, n: 1045175573, x: 24}, {e: lookup.Epoch{Time: 1057396147, Level: 20}, n: 1057396147, x: 19}, {e: lookup.Epoch{Time: 1048906375, Level: 18}, n: 1039616919, x: 25}, {e: lookup.Epoch{Time: 1074294831, Level: 20}, n: 1074294831, x: 19}, {e: lookup.Epoch{Time: 1088946052, Level: 1}, n: 1088917364, x: 14}, {e: lookup.Epoch{Time: 1112337595, Level: 17}, n: 1111008110, x: 22}, {e: lookup.Epoch{Time: 1099990284, Level: 5}, n: 1099968370, x: 15}, {e: lookup.Epoch{Time: 1087036441, Level: 16}, n: 1053967855, x: 25}, {e: lookup.Epoch{Time: 1069225185, Level: 8}, n: 1069224660, x: 10}, {e: lookup.Epoch{Time: 1057505479, Level: 9}, n: 1057505170, x: 14}, {e: lookup.Epoch{Time: 1072381377, Level: 12}, n: 1065950959, x: 22}, {e: lookup.Epoch{Time: 1093887139, Level: 8}, n: 1093863305, x: 14}, {e: lookup.Epoch{Time: 1082366510, Level: 24}, n: 1082366510, x: 23}, {e: lookup.Epoch{Time: 1103231132, Level: 14}, n: 1102292201, x: 22}, {e: lookup.Epoch{Time: 1094502355, Level: 3}, n: 1094324652, x: 18}, {e: lookup.Epoch{Time: 1068488344, Level: 12}, n: 1067577330, x: 19}, {e: lookup.Epoch{Time: 1050278233, Level: 12}, n: 1050278232, x: 11}, {e: lookup.Epoch{Time: 1047660768, Level: 5}, n: 1047652137, x: 17}, {e: lookup.Epoch{Time: 1060116167, Level: 11}, n: 1060114091, x: 12}, {e: lookup.Epoch{Time: 1068149392, Level: 21}, n: 1052074801, x: 24}, {e: lookup.Epoch{Time: 1081934120, Level: 6}, n: 1081933847, x: 8}, {e: lookup.Epoch{Time: 1107943693, Level: 16}, n: 1107096139, x: 25}, {e: lookup.Epoch{Time: 1131571649, Level: 9}, n: 1131570428, x: 11}, {e: lookup.Epoch{Time: 1123139367, Level: 0}, n: 1122912198, x: 20}, {e: lookup.Epoch{Time: 1121144423, Level: 6}, n: 1120568289, x: 20}, {e: lookup.Epoch{Time: 1089932411, Level: 17}, n: 1089932410, x: 16}, {e: lookup.Epoch{Time: 1104899012, Level: 22}, n: 1098978789, x: 22}, {e: lookup.Epoch{Time: 1094588059, Level: 21}, n: 1094588059, x: 20}, {e: lookup.Epoch{Time: 1114987438, Level: 24}, n: 1114987437, x: 23}, {e: lookup.Epoch{Time: 1084186305, Level: 7}, n: 1084186241, x: 6}, {e: lookup.Epoch{Time: 1058827111, Level: 8}, n: 1058826504, x: 9}, {e: lookup.Epoch{Time: 1090679810, Level: 12}, n: 1090616539, x: 17}, {e: lookup.Epoch{Time: 1084299475, Level: 23}, n: 1084299475, x: 22}} func TestGetNextLevel(t *testing.T) { diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index 65b72acec8..aa65183e33 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -189,9 +189,9 @@ func TestMockDbStoreNotFound(t *testing.T) { } func testIterator(t *testing.T, mock bool) { - var chunkcount int = 32 var i int var poc uint + chunkcount := 32 chunkkeys := NewAddressCollection(chunkcount) chunkkeysResults := NewAddressCollection(chunkcount) From 872370e3bc3c753937784e33e32d499ec277ab5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Mon, 25 Feb 2019 10:03:31 +0100 Subject: [PATCH 08/21] swarm/network/simulation: do not copy node mutex in UploadSnapshot (#19160) --- swarm/network/simulation/node.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/network/simulation/node.go b/swarm/network/simulation/node.go index 24afe51a41..4c3834de30 100644 --- a/swarm/network/simulation/node.go +++ b/swarm/network/simulation/node.go @@ -222,11 +222,11 @@ func (s *Simulation) UploadSnapshot(snapshotFile string, opts ...AddNodeOption) //the snapshot probably has the property EnableMsgEvents not set //just in case, set it to true! //(we need this to wait for messages before uploading) - for _, n := range snap.Nodes { - n.Node.Config.EnableMsgEvents = true - n.Node.Config.Services = s.serviceNames + for i := range snap.Nodes { + snap.Nodes[i].Node.Config.EnableMsgEvents = true + snap.Nodes[i].Node.Config.Services = s.serviceNames for _, o := range opts { - o(n.Node.Config) + o(snap.Nodes[i].Node.Config) } } From 7d881e45bdaad3cf1a301c7523a07026a479fd04 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 25 Feb 2019 11:34:08 +0100 Subject: [PATCH 09/21] rlp: added pooling of streams using sync (#19044) Prevents reallocation, improves performance --- rlp/decode.go | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index dbbe599597..e638c01eaf 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -26,6 +26,7 @@ import ( "math/big" "reflect" "strings" + "sync" ) var ( @@ -48,6 +49,10 @@ var ( errUintOverflow = errors.New("rlp: uint overflow") errNoPointer = errors.New("rlp: interface given to Decode must be a pointer") errDecodeIntoNil = errors.New("rlp: pointer given to Decode must not be nil") + + streamPool = sync.Pool{ + New: func() interface{} { return new(Stream) }, + } ) // Decoder is implemented by types that require custom RLP @@ -126,17 +131,24 @@ type Decoder interface { // // NewStream(r, limit).Decode(val) func Decode(r io.Reader, val interface{}) error { - // TODO: this could use a Stream from a pool. - return NewStream(r, 0).Decode(val) + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, 0) + return stream.Decode(val) } // DecodeBytes parses RLP data from b into val. // Please see the documentation of Decode for the decoding rules. // The input must contain exactly one value and no trailing data. func DecodeBytes(b []byte, val interface{}) error { - // TODO: this could use a Stream from a pool. r := bytes.NewReader(b) - if err := NewStream(r, uint64(len(b))).Decode(val); err != nil { + + stream := streamPool.Get().(*Stream) + defer streamPool.Put(stream) + + stream.Reset(r, uint64(len(b))) + if err := stream.Decode(val); err != nil { return err } if r.Len() > 0 { @@ -853,6 +865,7 @@ func (s *Stream) Reset(r io.Reader, inputLimit uint64) { if s.uintbuf == nil { s.uintbuf = make([]byte, 8) } + s.byteval = 0 } // Kind returns the kind and size of the next value in the From f5f742d1acab85c2287d694ea99aa92131f554d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 25 Feb 2019 12:40:33 +0200 Subject: [PATCH 10/21] containers/docker: nuke per the 1.8.0 deprecation note --- containers/docker/develop-alpine/Dockerfile | 14 -------------- containers/docker/develop-ubuntu/Dockerfile | 17 ----------------- containers/docker/master-alpine/Dockerfile | 14 -------------- containers/docker/master-ubuntu/Dockerfile | 17 ----------------- 4 files changed, 62 deletions(-) delete mode 100644 containers/docker/develop-alpine/Dockerfile delete mode 100644 containers/docker/develop-ubuntu/Dockerfile delete mode 100644 containers/docker/master-alpine/Dockerfile delete mode 100644 containers/docker/master-ubuntu/Dockerfile diff --git a/containers/docker/develop-alpine/Dockerfile b/containers/docker/develop-alpine/Dockerfile deleted file mode 100644 index 1f3e1112ec..0000000000 --- a/containers/docker/develop-alpine/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM alpine:3.7 - -RUN \ - apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 https://github.com/ethereum/go-ethereum && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /geth && \ - apk del go git make gcc musl-dev linux-headers && \ - rm -rf /go-ethereum && rm -rf /var/cache/apk/* - -EXPOSE 8545 -EXPOSE 30303 - -ENTRYPOINT ["/geth"] diff --git a/containers/docker/develop-ubuntu/Dockerfile b/containers/docker/develop-ubuntu/Dockerfile deleted file mode 100644 index 8c4fe9f595..0000000000 --- a/containers/docker/develop-ubuntu/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:xenial - -ENV PATH=/usr/lib/go-1.9/bin:$PATH - -RUN \ - apt-get update && apt-get upgrade -q -y && \ - apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \ - git clone --depth 1 https://github.com/ethereum/go-ethereum && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /geth && \ - apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \ - rm -rf /go-ethereum - -EXPOSE 8545 -EXPOSE 30303 - -ENTRYPOINT ["/geth"] diff --git a/containers/docker/master-alpine/Dockerfile b/containers/docker/master-alpine/Dockerfile deleted file mode 100644 index 8d4e7fe81a..0000000000 --- a/containers/docker/master-alpine/Dockerfile +++ /dev/null @@ -1,14 +0,0 @@ -FROM alpine:3.7 - -RUN \ - apk add --update go git make gcc musl-dev linux-headers ca-certificates && \ - git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /geth && \ - apk del go git make gcc musl-dev linux-headers && \ - rm -rf /go-ethereum && rm -rf /var/cache/apk/* - -EXPOSE 8545 -EXPOSE 30303 - -ENTRYPOINT ["/geth"] diff --git a/containers/docker/master-ubuntu/Dockerfile b/containers/docker/master-ubuntu/Dockerfile deleted file mode 100644 index 4cfc4f58c0..0000000000 --- a/containers/docker/master-ubuntu/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM ubuntu:xenial - -ENV PATH=/usr/lib/go-1.9/bin:$PATH - -RUN \ - apt-get update && apt-get upgrade -q -y && \ - apt-get install -y --no-install-recommends golang-1.9 git make gcc libc-dev ca-certificates && \ - git clone --depth 1 --branch release/1.8 https://github.com/ethereum/go-ethereum && \ - (cd go-ethereum && make geth) && \ - cp go-ethereum/build/bin/geth /geth && \ - apt-get remove -y golang-1.9 git make gcc libc-dev && apt autoremove -y && apt-get clean && \ - rm -rf /go-ethereum - -EXPOSE 8545 -EXPOSE 30303 - -ENTRYPOINT ["/geth"] From badaf43019dce961d6562b81d34de214e9e8ac0f Mon Sep 17 00:00:00 2001 From: Matthew Halpern Date: Mon, 25 Feb 2019 02:49:49 -0800 Subject: [PATCH 11/21] les: remove redundant type specifiers (#19091) --- les/fetcher_test.go | 11 ++++------- les/peer_test.go | 14 ++++++-------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/les/fetcher_test.go b/les/fetcher_test.go index 4ae7f6d04a..cc678c8c13 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -14,13 +14,10 @@ import ( ) func TestFetcherULCPeerSelector(t *testing.T) { - - var ( - id1 enode.ID = newNodeID(t).ID() - id2 enode.ID = newNodeID(t).ID() - id3 enode.ID = newNodeID(t).ID() - id4 enode.ID = newNodeID(t).ID() - ) + id1 := newNodeID(t).ID() + id2 := newNodeID(t).ID() + id3 := newNodeID(t).ID() + id4 := newNodeID(t).ID() ftn1 := &fetcherTreeNode{ hash: common.HexToHash("1"), diff --git a/les/peer_test.go b/les/peer_test.go index 0ba9cbfd52..a0e0a300cb 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -7,7 +7,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" ) @@ -25,8 +24,7 @@ var ( //ulc connects to trusted peer and send announceType=announceTypeSigned func TestPeerHandshakeSetAnnounceTypeToAnnounceTypeSignedForTrustedPeer(t *testing.T) { - - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() //peer to connect(on ulc side) p := peer{ @@ -74,7 +72,7 @@ func TestPeerHandshakeSetAnnounceTypeToAnnounceTypeSignedForTrustedPeer(t *testi } func TestPeerHandshakeAnnounceTypeSignedForTrustedPeersPeerNotInTrusted(t *testing.T) { - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() p := peer{ Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), version: protocol_version, @@ -118,7 +116,7 @@ func TestPeerHandshakeAnnounceTypeSignedForTrustedPeersPeerNotInTrusted(t *testi } func TestPeerHandshakeDefaultAllRequests(t *testing.T) { - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() s := generateLesServer() @@ -147,7 +145,7 @@ func TestPeerHandshakeDefaultAllRequests(t *testing.T) { } func TestPeerHandshakeServerSendOnlyAnnounceRequestsHeaders(t *testing.T) { - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() s := generateLesServer() s.onlyAnnounce = true @@ -181,7 +179,7 @@ func TestPeerHandshakeServerSendOnlyAnnounceRequestsHeaders(t *testing.T) { } } func TestPeerHandshakeClientReceiveOnlyAnnounceRequestsHeaders(t *testing.T) { - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() p := peer{ Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), @@ -212,7 +210,7 @@ func TestPeerHandshakeClientReceiveOnlyAnnounceRequestsHeaders(t *testing.T) { } func TestPeerHandshakeClientReturnErrorOnUselessPeer(t *testing.T) { - var id enode.ID = newNodeID(t).ID() + id := newNodeID(t).ID() p := peer{ Peer: p2p.NewPeer(id, "test peer", []p2p.Cap{}), From 340a53a98bd760fab229ea9daa70adddf7460a78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 26 Feb 2019 08:17:20 +0100 Subject: [PATCH 12/21] swarm/pss: fix data race on HandshakeController.symKeyIndex (#19162) * swarm/pss: fix data race on HandshakeController.symKeyIndex The HandshakeController.symKeyIndex map was accessed concurrently. Since insufficient test coverage the race is not detected every time. However, running TestClientHandshake a 100 times seems to be enough to reproduce the race. Note: I've chosen HandshakeController.lock to protect HandshakeController.symKeyIndex as that was already protected in a few functions by that lock. Additionally: - removed unused testStore - enabled tests in handshake_test.go as they pass - removed code duplication by adding getSymKey() * swarm/pss: fix a data race on HandshakeController.keyC * swarm/pss: fix data races with on Pss.symKeyPool --- swarm/pss/client/client_test.go | 16 ------- swarm/pss/handshake.go | 78 ++++++++++++++++++++++----------- swarm/pss/handshake_test.go | 5 +-- swarm/pss/keystore.go | 4 +- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 1c6f2e522d..1bd340cf04 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -23,7 +23,6 @@ import ( "fmt" "math/rand" "os" - "sync" "testing" "time" @@ -286,18 +285,3 @@ func newServices() adapters.Services { }, } } - -// copied from swarm/network/protocol_test_go -type testStore struct { - sync.Mutex - - values map[string][]byte -} - -func (t *testStore) Load(key string) ([]byte, error) { - return nil, nil -} - -func (t *testStore) Save(key string, v []byte) error { - return nil -} diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index bb67b51563..ec3bffa30a 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -106,6 +106,7 @@ func NewHandshakeParams() *HandshakeParams { type HandshakeController struct { pss *Pss keyC map[string]chan []string // adds a channel to report when a handshake succeeds + keyCMu sync.Mutex // protects keyC map lock sync.Mutex symKeyRequestTimeout time.Duration symKeyExpiryTimeout time.Duration @@ -165,9 +166,9 @@ func (ctl *HandshakeController) validKeys(pubkeyid string, topic *Topic, in bool for _, key := range *keystore { if key.limit <= key.count { - ctl.releaseKey(*key.symKeyID, topic) + ctl.releaseKeyNoLock(*key.symKeyID, topic) } else if !key.expiredAt.IsZero() && key.expiredAt.Before(now) { - ctl.releaseKey(*key.symKeyID, topic) + ctl.releaseKeyNoLock(*key.symKeyID, topic) } else { validkeys = append(validkeys, key.symKeyID) } @@ -205,15 +206,23 @@ func (ctl *HandshakeController) updateKeys(pubkeyid string, topic *Topic, in boo limit: limit, } *keystore = append(*keystore, storekey) + ctl.pss.mx.Lock() ctl.pss.symKeyPool[*storekey.symKeyID][*topic].protected = true + ctl.pss.mx.Unlock() } for i := 0; i < len(*keystore); i++ { ctl.symKeyIndex[*(*keystore)[i].symKeyID] = &((*keystore)[i]) } } -// Expire a symmetric key, making it elegible for garbage collection func (ctl *HandshakeController) releaseKey(symkeyid string, topic *Topic) bool { + ctl.lock.Lock() + defer ctl.lock.Unlock() + return ctl.releaseKeyNoLock(symkeyid, topic) +} + +// Expire a symmetric key, making it eligible for garbage collection +func (ctl *HandshakeController) releaseKeyNoLock(symkeyid string, topic *Topic) bool { if ctl.symKeyIndex[symkeyid] == nil { log.Debug("no symkey", "symkeyid", symkeyid) return false @@ -276,30 +285,49 @@ func (ctl *HandshakeController) clean() { } } +func (ctl *HandshakeController) getSymKey(symkeyid string) *handshakeKey { + ctl.lock.Lock() + defer ctl.lock.Unlock() + return ctl.symKeyIndex[symkeyid] +} + // Passed as a PssMsg handler for the topic handshake is activated on // Handles incoming key exchange messages and -// ccunts message usage by symmetric key (expiry limit control) +// counts message usage by symmetric key (expiry limit control) // Only returns error if key handler fails func (ctl *HandshakeController) handler(msg []byte, p *p2p.Peer, asymmetric bool, symkeyid string) error { - if !asymmetric { - if ctl.symKeyIndex[symkeyid] != nil { - if ctl.symKeyIndex[symkeyid].count >= ctl.symKeyIndex[symkeyid].limit { - return fmt.Errorf("discarding message using expired key: %s", symkeyid) + if asymmetric { + keymsg := &handshakeMsg{} + err := rlp.DecodeBytes(msg, keymsg) + if err == nil { + err := ctl.handleKeys(symkeyid, keymsg) + if err != nil { + log.Error("handlekeys fail", "error", err) } - ctl.symKeyIndex[symkeyid].count++ - log.Trace("increment symkey recv use", "symsymkeyid", symkeyid, "count", ctl.symKeyIndex[symkeyid].count, "limit", ctl.symKeyIndex[symkeyid].limit, "receiver", common.ToHex(crypto.FromECDSAPub(ctl.pss.PublicKey()))) + return err } return nil } - keymsg := &handshakeMsg{} - err := rlp.DecodeBytes(msg, keymsg) - if err == nil { - err := ctl.handleKeys(symkeyid, keymsg) - if err != nil { - log.Error("handlekeys fail", "error", err) - } - return err + return ctl.registerSymKeyUse(symkeyid) +} + +func (ctl *HandshakeController) registerSymKeyUse(symkeyid string) error { + ctl.lock.Lock() + defer ctl.lock.Unlock() + + symKey, ok := ctl.symKeyIndex[symkeyid] + if !ok { + return nil } + + if symKey.count >= symKey.limit { + return fmt.Errorf("symetric key expired (id: %s)", symkeyid) + } + symKey.count++ + + receiver := common.ToHex(crypto.FromECDSAPub(ctl.pss.PublicKey())) + log.Trace("increment symkey recv use", "symsymkeyid", symkeyid, "count", symKey.count, "limit", symKey.limit, "receiver", receiver) + return nil } @@ -417,6 +445,8 @@ func (ctl *HandshakeController) sendKey(pubkeyid string, topic *Topic, keycount // Enables callback for keys received from a key exchange request func (ctl *HandshakeController) alertHandshake(pubkeyid string, symkeys []string) chan []string { + ctl.keyCMu.Lock() + defer ctl.keyCMu.Unlock() if len(symkeys) > 0 { if _, ok := ctl.keyC[pubkeyid]; ok { ctl.keyC[pubkeyid] <- symkeys @@ -519,7 +549,7 @@ func (api *HandshakeAPI) GetHandshakeKeys(pubkeyid string, topic Topic, in bool, // Returns the amount of messages the specified symmetric key // is still valid for under the handshake scheme func (api *HandshakeAPI) GetHandshakeKeyCapacity(symkeyid string) (uint16, error) { - storekey := api.ctrl.symKeyIndex[symkeyid] + storekey := api.ctrl.getSymKey(symkeyid) if storekey == nil { return 0, fmt.Errorf("invalid symkey id %s", symkeyid) } @@ -529,7 +559,7 @@ func (api *HandshakeAPI) GetHandshakeKeyCapacity(symkeyid string) (uint16, error // Returns the byte representation of the public key in ascii hex // associated with the given symmetric key func (api *HandshakeAPI) GetHandshakePublicKey(symkeyid string) (string, error) { - storekey := api.ctrl.symKeyIndex[symkeyid] + storekey := api.ctrl.getSymKey(symkeyid) if storekey == nil { return "", fmt.Errorf("invalid symkey id %s", symkeyid) } @@ -555,12 +585,8 @@ func (api *HandshakeAPI) ReleaseHandshakeKey(pubkeyid string, topic Topic, symke // for message expiry control func (api *HandshakeAPI) SendSym(symkeyid string, topic Topic, msg hexutil.Bytes) (err error) { err = api.ctrl.pss.SendSym(symkeyid, topic, msg[:]) - if api.ctrl.symKeyIndex[symkeyid] != nil { - if api.ctrl.symKeyIndex[symkeyid].count >= api.ctrl.symKeyIndex[symkeyid].limit { - return errors.New("attempted send with expired key") - } - api.ctrl.symKeyIndex[symkeyid].count++ - log.Trace("increment symkey send use", "symkeyid", symkeyid, "count", api.ctrl.symKeyIndex[symkeyid].count, "limit", api.ctrl.symKeyIndex[symkeyid].limit, "receiver", common.ToHex(crypto.FromECDSAPub(api.ctrl.pss.PublicKey()))) + if otherErr := api.ctrl.registerSymKeyUse(symkeyid); otherErr != nil { + return otherErr } return err } diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index 895163f301..f4effc022d 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build foo - package pss import ( @@ -30,7 +28,6 @@ import ( // asymmetrical key exchange between two directly connected peers // full address, partial address (8 bytes) and empty address func TestHandshake(t *testing.T) { - t.Skip("handshakes are not adapted to current pss core code") t.Run("32", testHandshake) t.Run("8", testHandshake) t.Run("0", testHandshake) @@ -47,7 +44,7 @@ func testHandshake(t *testing.T) { // set up two nodes directly connected // (we are not testing pss routing here) - clients, err := setupNetwork(2) + clients, err := setupNetwork(2, true) if err != nil { t.Fatal(err) } diff --git a/swarm/pss/keystore.go b/swarm/pss/keystore.go index 510d21bcfd..5c44cb2453 100644 --- a/swarm/pss/keystore.go +++ b/swarm/pss/keystore.go @@ -210,6 +210,8 @@ func (ks *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage // - it is not marked as protected // - it is not in the incoming decryption cache func (ks *Pss) cleanKeys() (count int) { + ks.mx.Lock() + defer ks.mx.Unlock() for keyid, peertopics := range ks.symKeyPool { var expiredtopics []Topic for topic, psp := range peertopics { @@ -229,10 +231,8 @@ func (ks *Pss) cleanKeys() (count int) { } } for _, topic := range expiredtopics { - ks.mx.Lock() delete(ks.symKeyPool[keyid], topic) log.Trace("symkey cleanup deletion", "symkeyid", keyid, "topic", topic, "val", ks.symKeyPool[keyid]) - ks.mx.Unlock() count++ } } From c83ba9e7941a0a97e20922e04ef403e52d885017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 26 Feb 2019 10:02:05 +0100 Subject: [PATCH 13/21] swarm/storage/localstore: fix tests for windows os (#19161) --- swarm/storage/localstore/localstore_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 6b48d54e98..6954b139a2 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "math/rand" "os" + "runtime" "sort" "strconv" "sync" @@ -34,6 +35,26 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) +func init() { + // Some of the tests in localstore package rely on the same ordering of + // items uploaded or accessed compared to the ordering of items in indexes + // that contain StoreTimestamp or AccessTimestamp in keys. In tests + // where the same order is required from the database as the order + // in which chunks are put or accessed, if the StoreTimestamp or + // AccessTimestamp are the same for two or more sequential items + // their order in database will be based on the chunk address value, + // in which case the ordering of items/chunks stored in a test slice + // will not be the same. To ensure the same ordering in database on such + // indexes on windows systems, an additional short sleep is added to + // the now function. + if runtime.GOOS == "windows" { + setNow(func() int64 { + time.Sleep(time.Microsecond) + return time.Now().UTC().UnixNano() + }) + } +} + // TestDB validates if the chunk can be uploaded and // correctly retrieved. func TestDB(t *testing.T) { From 647692fbfe5a524c11211a7d4c5c4e1d78f4048f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 26 Feb 2019 10:31:37 +0200 Subject: [PATCH 14/21] build: bump PPA builders to Go 1.11 --- build/ci-notes.md | 6 +++--- build/deb/ethereum-swarm/deb.control | 2 +- build/deb/ethereum-swarm/deb.rules | 2 +- build/deb/ethereum/deb.control | 2 +- build/deb/ethereum/deb.rules | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build/ci-notes.md b/build/ci-notes.md index ba27cdb1d7..13e1fd2307 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -23,18 +23,18 @@ variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, our PPA depends on the ~gophers/ubuntu/archive PPA. Our source package build-depends on -golang-1.10, which is co-installable alongside the regular golang package. PPA dependencies +golang-1.11, which is co-installable alongside the regular golang package. PPA dependencies can be edited at https://launchpad.net/%7Eethereum/+archive/ubuntu/ethereum/+edit-dependencies ## Building Packages Locally (for testing) You need to run Ubuntu to do test packaging. -Add the gophers PPA and install Go 1.10 and Debian packaging tools: +Add the gophers PPA and install Go 1.11 and Debian packaging tools: $ sudo apt-add-repository ppa:gophers/ubuntu/archive $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko + $ sudo apt-get install build-essential golang-1.11 devscripts debhelper python-bzrlib python-paramiko Create the source packages: diff --git a/build/deb/ethereum-swarm/deb.control b/build/deb/ethereum-swarm/deb.control index 8cd325bf55..b0ced141b6 100644 --- a/build/deb/ethereum-swarm/deb.control +++ b/build/deb/ethereum-swarm/deb.control @@ -2,7 +2,7 @@ Source: {{.Name}} Section: science Priority: extra Maintainer: {{.Author}} -Build-Depends: debhelper (>= 8.0.0), golang-1.10 +Build-Depends: debhelper (>= 8.0.0), golang-1.11 Standards-Version: 3.9.5 Homepage: https://ethereum.org Vcs-Git: git://github.com/ethereum/go-ethereum.git diff --git a/build/deb/ethereum-swarm/deb.rules b/build/deb/ethereum-swarm/deb.rules index 7f286569ea..98e2ab030e 100644 --- a/build/deb/ethereum-swarm/deb.rules +++ b/build/deb/ethereum-swarm/deb.rules @@ -5,7 +5,7 @@ #export DH_VERBOSE=1 override_dh_auto_build: - build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: diff --git a/build/deb/ethereum/deb.control b/build/deb/ethereum/deb.control index defb106fe3..018067a19d 100644 --- a/build/deb/ethereum/deb.control +++ b/build/deb/ethereum/deb.control @@ -2,7 +2,7 @@ Source: {{.Name}} Section: science Priority: extra Maintainer: {{.Author}} -Build-Depends: debhelper (>= 8.0.0), golang-1.10 +Build-Depends: debhelper (>= 8.0.0), golang-1.11 Standards-Version: 3.9.5 Homepage: https://ethereum.org Vcs-Git: git://github.com/ethereum/go-ethereum.git diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 7f286569ea..98e2ab030e 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -5,7 +5,7 @@ #export DH_VERBOSE=1 override_dh_auto_build: - build/env.sh /usr/lib/go-1.10/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + build/env.sh /usr/lib/go-1.11/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: From c2b33a117f686069e36d58d937e1c75d72d7b94c Mon Sep 17 00:00:00 2001 From: Roc Yu Date: Tue, 26 Feb 2019 19:19:43 +0800 Subject: [PATCH 15/21] graphql: fix typos in comments (#19041) --- graphql/grahpql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/graphql/grahpql.go b/graphql/grahpql.go index 1eca789567..68d36f9977 100644 --- a/graphql/grahpql.go +++ b/graphql/grahpql.go @@ -127,7 +127,7 @@ func (l *Log) Data(ctx context.Context) hexutil.Bytes { return hexutil.Bytes(l.log.Data) } -// Transactionn represents an Ethereum transaction. +// Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { backend *eth.EthAPIBackend @@ -916,7 +916,7 @@ func (r *Resolver) EstimateGas(ctx context.Context, args struct { return gas, err } -// FilterCritera encapsulates the arguments to `logs` on the root resolver object. +// FilterCriteria encapsulates the arguments to `logs` on the root resolver object. type FilterCriteria struct { FromBlock *hexutil.Uint64 // beginning of the queried range, nil means genesis block ToBlock *hexutil.Uint64 // end of the range, nil means latest block From c2003ed63b975c6318e4dd7e65b69c60777b0ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 26 Feb 2019 12:32:48 +0100 Subject: [PATCH 16/21] les, les/flowcontrol: improved request serving and flow control (#18230) This change - implements concurrent LES request serving even for a single peer. - replaces the request cost estimation method with a cost table based on benchmarks which gives much more consistent results. Until now the allowed number of light peers was just a guess which probably contributed a lot to the fluctuating quality of available service. Everything related to request cost is implemented in a single object, the 'cost tracker'. It uses a fixed cost table with a global 'correction factor'. Benchmark code is included and can be run at any time to adapt costs to low-level implementation changes. - reimplements flowcontrol.ClientManager in a cleaner and more efficient way, with added capabilities: There is now control over bandwidth, which allows using the flow control parameters for client prioritization. Target utilization over 100 percent is now supported to model concurrent request processing. Total serving bandwidth is reduced during block processing to prevent database contention. - implements an RPC API for the LES servers allowing server operators to assign priority bandwidth to certain clients and change prioritized status even while the client is connected. The new API is meant for cases where server operators charge for LES using an off-protocol mechanism. - adds a unit test for the new client manager. - adds an end-to-end test using the network simulator that tests bandwidth control functions through the new API. --- cmd/geth/main.go | 2 + cmd/geth/usage.go | 2 + cmd/utils/flags.go | 14 +- core/blockchain.go | 11 + eth/backend.go | 5 + eth/config.go | 8 +- eth/gen_config.go | 12 + les/api.go | 454 +++++++++++++++++ les/api_test.go | 525 ++++++++++++++++++++ les/backend.go | 3 +- les/benchmark.go | 353 +++++++++++++ les/costtracker.go | 388 +++++++++++++++ les/distributor.go | 10 +- les/distributor_test.go | 6 +- les/fetcher.go | 3 +- les/flowcontrol/control.go | 369 ++++++++++---- les/flowcontrol/logger.go | 65 +++ les/flowcontrol/manager.go | 493 ++++++++++++------ les/flowcontrol/manager_test.go | 123 +++++ les/freeclient.go | 95 +++- les/freeclient_test.go | 54 +- les/handler.go | 797 ++++++++++++++++-------------- les/helper_test.go | 23 +- les/odr.go | 2 +- les/odr_requests.go | 2 - les/peer.go | 254 +++++++--- les/peer_test.go | 15 +- les/protocol.go | 24 +- les/randselect.go | 1 - les/retrieve.go | 2 - les/server.go | 301 ++++------- les/serverpool.go | 1 - les/servingqueue.go | 261 ++++++++++ les/txrelay.go | 10 +- les/ulc_test.go | 3 +- light/lightchain.go | 2 + light/odr.go | 2 - p2p/simulations/adapters/exec.go | 6 +- p2p/simulations/adapters/types.go | 3 + 39 files changed, 3705 insertions(+), 999 deletions(-) create mode 100644 les/api.go create mode 100644 les/api_test.go create mode 100644 les/benchmark.go create mode 100644 les/costtracker.go create mode 100644 les/flowcontrol/logger.go create mode 100644 les/flowcontrol/manager_test.go create mode 100644 les/servingqueue.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 17bf438e2a..a331abc9fd 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -93,6 +93,8 @@ var ( utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.LightServFlag, + utils.LightBandwidthInFlag, + utils.LightBandwidthOutFlag, utils.LightPeersFlag, utils.LightKDFFlag, utils.WhitelistFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index a262037168..0338e447e4 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -81,6 +81,8 @@ var AppHelpFlagGroups = []flagGroup{ utils.EthStatsURLFlag, utils.IdentityFlag, utils.LightServFlag, + utils.LightBandwidthInFlag, + utils.LightBandwidthOutFlag, utils.LightPeersFlag, utils.LightKDFFlag, utils.WhitelistFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5b8ebb481f..4db59097d7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -199,9 +199,19 @@ var ( } LightServFlag = cli.IntFlag{ Name: "lightserv", - Usage: "Maximum percentage of time allowed for serving LES requests (0-90)", + Usage: "Maximum percentage of time allowed for serving LES requests (multi-threaded processing allows values over 100)", Value: 0, } + LightBandwidthInFlag = cli.IntFlag{ + Name: "lightbwin", + Usage: "Incoming bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)", + Value: 1000, + } + LightBandwidthOutFlag = cli.IntFlag{ + Name: "lightbwout", + Usage: "Outgoing bandwidth limit for light server (1000 bytes/sec, 0 = unlimited)", + Value: 5000, + } LightPeersFlag = cli.IntFlag{ Name: "lightpeers", Usage: "Maximum number of LES client peers", @@ -1305,6 +1315,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(LightServFlag.Name) { cfg.LightServ = ctx.GlobalInt(LightServFlag.Name) } + cfg.LightBandwidthIn = ctx.GlobalInt(LightBandwidthInFlag.Name) + cfg.LightBandwidthOut = ctx.GlobalInt(LightBandwidthOutFlag.Name) if ctx.GlobalIsSet(LightPeersFlag.Name) { cfg.LightPeers = ctx.GlobalInt(LightPeersFlag.Name) } diff --git a/core/blockchain.go b/core/blockchain.go index d6dad27991..7b4f4b3038 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -111,6 +111,7 @@ type BlockChain struct { chainSideFeed event.Feed chainHeadFeed event.Feed logsFeed event.Feed + blockProcFeed event.Feed scope event.SubscriptionScope genesisBlock *types.Block @@ -1090,6 +1091,10 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { if len(chain) == 0 { return 0, nil } + + bc.blockProcFeed.Send(true) + defer bc.blockProcFeed.Send(false) + // Remove already known canon-blocks var ( block, prev *types.Block @@ -1725,3 +1730,9 @@ func (bc *BlockChain) SubscribeChainSideEvent(ch chan<- ChainSideEvent) event.Su func (bc *BlockChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return bc.scope.Track(bc.logsFeed.Subscribe(ch)) } + +// SubscribeBlockProcessingEvent registers a subscription of bool where true means +// block processing has started while false means it has stopped. +func (bc *BlockChain) SubscribeBlockProcessingEvent(ch chan<- bool) event.Subscription { + return bc.scope.Track(bc.blockProcFeed.Subscribe(ch)) +} diff --git a/eth/backend.go b/eth/backend.go index 6710e45131..cccb5993f1 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -54,6 +54,7 @@ import ( type LesServer interface { Start(srvr *p2p.Server) Stop() + APIs() []rpc.API Protocols() []p2p.Protocol SetBloomBitsIndexer(bbIndexer *core.ChainIndexer) } @@ -267,6 +268,10 @@ func CreateConsensusEngine(ctx *node.ServiceContext, chainConfig *params.ChainCo func (s *Ethereum) APIs() []rpc.API { apis := ethapi.GetAPIs(s.APIBackend) + // Append any APIs exposed explicitly by the les server + if s.lesServer != nil { + apis = append(apis, s.lesServer.APIs()...) + } // Append any APIs exposed explicitly by the consensus engine apis = append(apis, s.engine.APIs(s.BlockChain())...) diff --git a/eth/config.go b/eth/config.go index aca9b5e68b..740e6825b0 100644 --- a/eth/config.go +++ b/eth/config.go @@ -98,9 +98,11 @@ type Config struct { Whitelist map[uint64]common.Hash `toml:"-"` // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - OnlyAnnounce bool // Maximum number of LES client peers + LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests + LightBandwidthIn int `toml:",omitempty"` // Incoming bandwidth limit for light servers + LightBandwidthOut int `toml:",omitempty"` // Outgoing bandwidth limit for light servers + LightPeers int `toml:",omitempty"` // Maximum number of LES client peers + OnlyAnnounce bool // Maximum number of LES client peers // Ultra Light client options ULC *ULCConfig `toml:",omitempty"` diff --git a/eth/gen_config.go b/eth/gen_config.go index e05b963abe..30ff8b6e13 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -24,6 +24,8 @@ func (c Config) MarshalTOML() (interface{}, error) { SyncMode downloader.SyncMode NoPruning bool LightServ int `toml:",omitempty"` + LightBandwidthIn int `toml:",omitempty"` + LightBandwidthOut int `toml:",omitempty"` LightPeers int `toml:",omitempty"` OnlyAnnounce bool ULC *ULCConfig `toml:",omitempty"` @@ -55,6 +57,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.SyncMode = c.SyncMode enc.NoPruning = c.NoPruning enc.LightServ = c.LightServ + enc.LightBandwidthIn = c.LightBandwidthIn + enc.LightBandwidthOut = c.LightBandwidthOut enc.LightPeers = c.LightPeers enc.OnlyAnnounce = c.OnlyAnnounce enc.ULC = c.ULC @@ -91,6 +95,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { SyncMode *downloader.SyncMode NoPruning *bool LightServ *int `toml:",omitempty"` + LightBandwidthIn *int `toml:",omitempty"` + LightBandwidthOut *int `toml:",omitempty"` LightPeers *int `toml:",omitempty"` OnlyAnnounce *bool ULC *ULCConfig `toml:",omitempty"` @@ -135,6 +141,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightServ != nil { c.LightServ = *dec.LightServ } + if dec.LightBandwidthIn != nil { + c.LightBandwidthIn = *dec.LightBandwidthIn + } + if dec.LightBandwidthOut != nil { + c.LightBandwidthOut = *dec.LightBandwidthOut + } if dec.LightPeers != nil { c.LightPeers = *dec.LightPeers } diff --git a/les/api.go b/les/api.go new file mode 100644 index 0000000000..a933cbd068 --- /dev/null +++ b/les/api.go @@ -0,0 +1,454 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rpc" +) + +var ( + ErrMinCap = errors.New("capacity too small") + ErrTotalCap = errors.New("total capacity exceeded") + ErrUnknownBenchmarkType = errors.New("unknown benchmark type") + + dropCapacityDelay = time.Second // delay applied to decreasing capacity changes +) + +// PrivateLightServerAPI provides an API to access the LES light server. +// It offers only methods that operate on public data that is freely available to anyone. +type PrivateLightServerAPI struct { + server *LesServer +} + +// NewPrivateLightServerAPI creates a new LES light server API. +func NewPrivateLightServerAPI(server *LesServer) *PrivateLightServerAPI { + return &PrivateLightServerAPI{ + server: server, + } +} + +// TotalCapacity queries total available capacity for all clients +func (api *PrivateLightServerAPI) TotalCapacity() hexutil.Uint64 { + return hexutil.Uint64(api.server.priorityClientPool.totalCapacity()) +} + +// SubscribeTotalCapacity subscribes to changed total capacity events. +// If onlyUnderrun is true then notification is sent only if the total capacity +// drops under the total capacity of connected priority clients. +// +// Note: actually applying decreasing total capacity values is delayed while the +// notification is sent instantly. This allows lowering the capacity of a priority client +// or choosing which one to drop before the system drops some of them automatically. +func (api *PrivateLightServerAPI) SubscribeTotalCapacity(ctx context.Context, onlyUnderrun bool) (*rpc.Subscription, error) { + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported + } + rpcSub := notifier.CreateSubscription() + api.server.priorityClientPool.subscribeTotalCapacity(&tcSubscription{notifier, rpcSub, onlyUnderrun}) + return rpcSub, nil +} + +type ( + // tcSubscription represents a total capacity subscription + tcSubscription struct { + notifier *rpc.Notifier + rpcSub *rpc.Subscription + onlyUnderrun bool + } + tcSubs map[*tcSubscription]struct{} +) + +// send sends a changed total capacity event to the subscribers +func (s tcSubs) send(tc uint64, underrun bool) { + for sub := range s { + select { + case <-sub.rpcSub.Err(): + delete(s, sub) + case <-sub.notifier.Closed(): + delete(s, sub) + default: + if underrun || !sub.onlyUnderrun { + sub.notifier.Notify(sub.rpcSub.ID, tc) + } + } + } +} + +// MinimumCapacity queries minimum assignable capacity for a single client +func (api *PrivateLightServerAPI) MinimumCapacity() hexutil.Uint64 { + return hexutil.Uint64(minCapacity) +} + +// FreeClientCapacity queries the capacity provided for free clients +func (api *PrivateLightServerAPI) FreeClientCapacity() hexutil.Uint64 { + return hexutil.Uint64(api.server.freeClientCap) +} + +// SetClientCapacity sets the priority capacity assigned to a given client. +// If the assigned capacity is bigger than zero then connection is always +// guaranteed. The sum of capacity assigned to priority clients can not exceed +// the total available capacity. +// +// Note: assigned capacity can be changed while the client is connected with +// immediate effect. +func (api *PrivateLightServerAPI) SetClientCapacity(id enode.ID, cap uint64) error { + if cap != 0 && cap < minCapacity { + return ErrMinCap + } + return api.server.priorityClientPool.setClientCapacity(id, cap) +} + +// GetClientCapacity returns the capacity assigned to a given client +func (api *PrivateLightServerAPI) GetClientCapacity(id enode.ID) hexutil.Uint64 { + api.server.priorityClientPool.lock.Lock() + defer api.server.priorityClientPool.lock.Unlock() + + return hexutil.Uint64(api.server.priorityClientPool.clients[id].cap) +} + +// clientPool is implemented by both the free and priority client pools +type clientPool interface { + peerSetNotify + setLimits(count int, totalCap uint64) +} + +// priorityClientPool stores information about prioritized clients +type priorityClientPool struct { + lock sync.Mutex + child clientPool + ps *peerSet + clients map[enode.ID]priorityClientInfo + totalCap, totalCapAnnounced uint64 + totalConnectedCap, freeClientCap uint64 + maxPeers, priorityCount int + + subs tcSubs + updateSchedule []scheduledUpdate + scheduleCounter uint64 +} + +// scheduledUpdate represents a delayed total capacity update +type scheduledUpdate struct { + time mclock.AbsTime + totalCap, id uint64 +} + +// priorityClientInfo entries exist for all prioritized clients and currently connected non-priority clients +type priorityClientInfo struct { + cap uint64 // zero for non-priority clients + connected bool + peer *peer +} + +// newPriorityClientPool creates a new priority client pool +func newPriorityClientPool(freeClientCap uint64, ps *peerSet, child clientPool) *priorityClientPool { + return &priorityClientPool{ + clients: make(map[enode.ID]priorityClientInfo), + freeClientCap: freeClientCap, + ps: ps, + child: child, + } +} + +// registerPeer is called when a new client is connected. If the client has no +// priority assigned then it is passed to the child pool which may either keep it +// or disconnect it. +// +// Note: priorityClientPool also stores a record about free clients while they are +// connected in order to be able to assign priority to them later. +func (v *priorityClientPool) registerPeer(p *peer) { + v.lock.Lock() + defer v.lock.Unlock() + + id := p.ID() + c := v.clients[id] + if c.connected { + return + } + if c.cap == 0 && v.child != nil { + v.child.registerPeer(p) + } + if c.cap != 0 && v.totalConnectedCap+c.cap > v.totalCap { + go v.ps.Unregister(p.id) + return + } + + c.connected = true + c.peer = p + v.clients[id] = c + if c.cap != 0 { + v.priorityCount++ + v.totalConnectedCap += c.cap + if v.child != nil { + v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) + } + p.updateCapacity(c.cap) + } +} + +// unregisterPeer is called when a client is disconnected. If the client has no +// priority assigned then it is also removed from the child pool. +func (v *priorityClientPool) unregisterPeer(p *peer) { + v.lock.Lock() + defer v.lock.Unlock() + + id := p.ID() + c := v.clients[id] + if !c.connected { + return + } + if c.cap != 0 { + c.connected = false + v.clients[id] = c + v.priorityCount-- + v.totalConnectedCap -= c.cap + if v.child != nil { + v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) + } + } else { + if v.child != nil { + v.child.unregisterPeer(p) + } + delete(v.clients, id) + } +} + +// setLimits updates the allowed peer count and total capacity of the priority +// client pool. Since the free client pool is a child of the priority pool the +// remaining peer count and capacity is assigned to the free pool by calling its +// own setLimits function. +// +// Note: a decreasing change of the total capacity is applied with a delay. +func (v *priorityClientPool) setLimits(count int, totalCap uint64) { + v.lock.Lock() + defer v.lock.Unlock() + + v.totalCapAnnounced = totalCap + if totalCap > v.totalCap { + v.setLimitsNow(count, totalCap) + v.subs.send(totalCap, false) + return + } + v.setLimitsNow(count, v.totalCap) + if totalCap < v.totalCap { + v.subs.send(totalCap, totalCap < v.totalConnectedCap) + for i, s := range v.updateSchedule { + if totalCap >= s.totalCap { + s.totalCap = totalCap + v.updateSchedule = v.updateSchedule[:i+1] + return + } + } + v.updateSchedule = append(v.updateSchedule, scheduledUpdate{time: mclock.Now() + mclock.AbsTime(dropCapacityDelay), totalCap: totalCap}) + if len(v.updateSchedule) == 1 { + v.scheduleCounter++ + id := v.scheduleCounter + v.updateSchedule[0].id = id + time.AfterFunc(dropCapacityDelay, func() { v.checkUpdate(id) }) + } + } else { + v.updateSchedule = nil + } +} + +// checkUpdate performs the next scheduled update if possible and schedules +// the one after that +func (v *priorityClientPool) checkUpdate(id uint64) { + v.lock.Lock() + defer v.lock.Unlock() + + if len(v.updateSchedule) == 0 || v.updateSchedule[0].id != id { + return + } + v.setLimitsNow(v.maxPeers, v.updateSchedule[0].totalCap) + v.updateSchedule = v.updateSchedule[1:] + if len(v.updateSchedule) != 0 { + v.scheduleCounter++ + id := v.scheduleCounter + v.updateSchedule[0].id = id + dt := time.Duration(v.updateSchedule[0].time - mclock.Now()) + time.AfterFunc(dt, func() { v.checkUpdate(id) }) + } +} + +// setLimits updates the allowed peer count and total capacity immediately +func (v *priorityClientPool) setLimitsNow(count int, totalCap uint64) { + if v.priorityCount > count || v.totalConnectedCap > totalCap { + for id, c := range v.clients { + if c.connected { + c.connected = false + v.totalConnectedCap -= c.cap + v.priorityCount-- + v.clients[id] = c + go v.ps.Unregister(c.peer.id) + if v.priorityCount <= count && v.totalConnectedCap <= totalCap { + break + } + } + } + } + v.maxPeers = count + v.totalCap = totalCap + if v.child != nil { + v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) + } +} + +// totalCapacity queries total available capacity for all clients +func (v *priorityClientPool) totalCapacity() uint64 { + v.lock.Lock() + defer v.lock.Unlock() + + return v.totalCapAnnounced +} + +// subscribeTotalCapacity subscribes to changed total capacity events +func (v *priorityClientPool) subscribeTotalCapacity(sub *tcSubscription) { + v.lock.Lock() + defer v.lock.Unlock() + + v.subs[sub] = struct{}{} +} + +// setClientCapacity sets the priority capacity assigned to a given client +func (v *priorityClientPool) setClientCapacity(id enode.ID, cap uint64) error { + v.lock.Lock() + defer v.lock.Unlock() + + c := v.clients[id] + if c.cap == cap { + return nil + } + if c.connected { + if v.totalConnectedCap+cap > v.totalCap+c.cap { + return ErrTotalCap + } + if c.cap == 0 { + if v.child != nil { + v.child.unregisterPeer(c.peer) + } + v.priorityCount++ + } + if cap == 0 { + v.priorityCount-- + } + v.totalConnectedCap += cap - c.cap + if v.child != nil { + v.child.setLimits(v.maxPeers-v.priorityCount, v.totalCap-v.totalConnectedCap) + } + if cap == 0 { + if v.child != nil { + v.child.registerPeer(c.peer) + } + c.peer.updateCapacity(v.freeClientCap) + } else { + c.peer.updateCapacity(cap) + } + } + if cap != 0 || c.connected { + c.cap = cap + v.clients[id] = c + } else { + delete(v.clients, id) + } + return nil +} + +// Benchmark runs a request performance benchmark with a given set of measurement setups +// in multiple passes specified by passCount. The measurement time for each setup in each +// pass is specified in milliseconds by length. +// +// Note: measurement time is adjusted for each pass depending on the previous ones. +// Therefore a controlled total measurement time is achievable in multiple passes. +func (api *PrivateLightServerAPI) Benchmark(setups []map[string]interface{}, passCount, length int) ([]map[string]interface{}, error) { + benchmarks := make([]requestBenchmark, len(setups)) + for i, setup := range setups { + if t, ok := setup["type"].(string); ok { + getInt := func(field string, def int) int { + if value, ok := setup[field].(float64); ok { + return int(value) + } + return def + } + getBool := func(field string, def bool) bool { + if value, ok := setup[field].(bool); ok { + return value + } + return def + } + switch t { + case "header": + benchmarks[i] = &benchmarkBlockHeaders{ + amount: getInt("amount", 1), + skip: getInt("skip", 1), + byHash: getBool("byHash", false), + reverse: getBool("reverse", false), + } + case "body": + benchmarks[i] = &benchmarkBodiesOrReceipts{receipts: false} + case "receipts": + benchmarks[i] = &benchmarkBodiesOrReceipts{receipts: true} + case "proof": + benchmarks[i] = &benchmarkProofsOrCode{code: false} + case "code": + benchmarks[i] = &benchmarkProofsOrCode{code: true} + case "cht": + benchmarks[i] = &benchmarkHelperTrie{ + bloom: false, + reqCount: getInt("amount", 1), + } + case "bloom": + benchmarks[i] = &benchmarkHelperTrie{ + bloom: true, + reqCount: getInt("amount", 1), + } + case "txSend": + benchmarks[i] = &benchmarkTxSend{} + case "txStatus": + benchmarks[i] = &benchmarkTxStatus{} + default: + return nil, ErrUnknownBenchmarkType + } + } else { + return nil, ErrUnknownBenchmarkType + } + } + rs := api.server.protocolManager.runBenchmark(benchmarks, passCount, time.Millisecond*time.Duration(length)) + result := make([]map[string]interface{}, len(setups)) + for i, r := range rs { + res := make(map[string]interface{}) + if r.err == nil { + res["totalCount"] = r.totalCount + res["avgTime"] = r.avgTime + res["maxInSize"] = r.maxInSize + res["maxOutSize"] = r.maxOutSize + } else { + res["error"] = r.err.Error() + } + result[i] = res + } + return result, nil +} diff --git a/les/api_test.go b/les/api_test.go new file mode 100644 index 0000000000..cec9459625 --- /dev/null +++ b/les/api_test.go @@ -0,0 +1,525 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "context" + "errors" + "flag" + "fmt" + "io/ioutil" + "math/rand" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rpc" + colorable "github.com/mattn/go-colorable" +) + +/* +This test is not meant to be a part of the automatic testing process because it +runs for a long time and also requires a large database in order to do a meaningful +request performance test. When testServerDataDir is empty, the test is skipped. +*/ + +const ( + testServerDataDir = "" // should always be empty on the master branch + testServerCapacity = 200 + testMaxClients = 10 + testTolerance = 0.1 + minRelCap = 0.2 +) + +func TestCapacityAPI3(t *testing.T) { + testCapacityAPI(t, 3) +} + +func TestCapacityAPI6(t *testing.T) { + testCapacityAPI(t, 6) +} + +func TestCapacityAPI10(t *testing.T) { + testCapacityAPI(t, 10) +} + +// testCapacityAPI runs an end-to-end simulation test connecting one server with +// a given number of clients. It sets different priority capacities to all clients +// except a randomly selected one which runs in free client mode. All clients send +// similar requests at the maximum allowed rate and the test verifies whether the +// ratio of processed requests is close enough to the ratio of assigned capacities. +// Running multiple rounds with different settings ensures that changing capacity +// while connected and going back and forth between free and priority mode with +// the supplied API calls is also thoroughly tested. +func testCapacityAPI(t *testing.T, clientCount int) { + if testServerDataDir == "" { + // Skip test if no data dir specified + return + } + + for !testSim(t, 1, clientCount, []string{testServerDataDir}, nil, func(ctx context.Context, net *simulations.Network, servers []*simulations.Node, clients []*simulations.Node) bool { + if len(servers) != 1 { + t.Fatalf("Invalid number of servers: %d", len(servers)) + } + server := servers[0] + + clientRpcClients := make([]*rpc.Client, len(clients)) + + serverRpcClient, err := server.Client() + if err != nil { + t.Fatalf("Failed to obtain rpc client: %v", err) + } + headNum, headHash := getHead(ctx, t, serverRpcClient) + totalCap := getTotalCap(ctx, t, serverRpcClient) + minCap := getMinCap(ctx, t, serverRpcClient) + testCap := totalCap * 3 / 4 + fmt.Printf("Server testCap: %d minCap: %d head number: %d head hash: %064x\n", testCap, minCap, headNum, headHash) + reqMinCap := uint64(float64(testCap) * minRelCap / (minRelCap + float64(len(clients)-1))) + if minCap > reqMinCap { + t.Fatalf("Minimum client capacity (%d) bigger than required minimum for this test (%d)", minCap, reqMinCap) + } + + freeIdx := rand.Intn(len(clients)) + freeCap := getFreeCap(ctx, t, serverRpcClient) + + for i, client := range clients { + var err error + clientRpcClients[i], err = client.Client() + if err != nil { + t.Fatalf("Failed to obtain rpc client: %v", err) + } + + fmt.Println("connecting client", i) + if i != freeIdx { + setCapacity(ctx, t, serverRpcClient, client.ID(), testCap/uint64(len(clients))) + } + net.Connect(client.ID(), server.ID()) + + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout") + default: + } + num, hash := getHead(ctx, t, clientRpcClients[i]) + if num == headNum && hash == headHash { + fmt.Println("client", i, "synced") + break + } + time.Sleep(time.Millisecond * 200) + } + } + + var wg sync.WaitGroup + stop := make(chan struct{}) + + reqCount := make([]uint64, len(clientRpcClients)) + + for i, c := range clientRpcClients { + wg.Add(1) + i, c := i, c + go func() { + queue := make(chan struct{}, 100) + var count uint64 + for { + select { + case queue <- struct{}{}: + select { + case <-stop: + wg.Done() + return + case <-ctx.Done(): + wg.Done() + return + default: + wg.Add(1) + go func() { + ok := testRequest(ctx, t, c) + wg.Done() + <-queue + if ok { + count++ + atomic.StoreUint64(&reqCount[i], count) + } + }() + } + case <-stop: + wg.Done() + return + case <-ctx.Done(): + wg.Done() + return + } + } + }() + } + + processedSince := func(start []uint64) []uint64 { + res := make([]uint64, len(reqCount)) + for i := range reqCount { + res[i] = atomic.LoadUint64(&reqCount[i]) + if start != nil { + res[i] -= start[i] + } + } + return res + } + + weights := make([]float64, len(clients)) + for c := 0; c < 5; c++ { + setCapacity(ctx, t, serverRpcClient, clients[freeIdx].ID(), freeCap) + freeIdx = rand.Intn(len(clients)) + var sum float64 + for i := range clients { + if i == freeIdx { + weights[i] = 0 + } else { + weights[i] = rand.Float64()*(1-minRelCap) + minRelCap + } + sum += weights[i] + } + for i, client := range clients { + weights[i] *= float64(testCap-freeCap-100) / sum + capacity := uint64(weights[i]) + if i != freeIdx && capacity < getCapacity(ctx, t, serverRpcClient, client.ID()) { + setCapacity(ctx, t, serverRpcClient, client.ID(), capacity) + } + } + setCapacity(ctx, t, serverRpcClient, clients[freeIdx].ID(), 0) + for i, client := range clients { + capacity := uint64(weights[i]) + if i != freeIdx && capacity > getCapacity(ctx, t, serverRpcClient, client.ID()) { + setCapacity(ctx, t, serverRpcClient, client.ID(), capacity) + } + } + weights[freeIdx] = float64(freeCap) + for i := range clients { + weights[i] /= float64(testCap) + } + + time.Sleep(flowcontrol.DecParamDelay) + fmt.Println("Starting measurement") + fmt.Printf("Relative weights:") + for i := range clients { + fmt.Printf(" %f", weights[i]) + } + fmt.Println() + start := processedSince(nil) + for { + select { + case <-ctx.Done(): + t.Fatalf("Timeout") + default: + } + + totalCap = getTotalCap(ctx, t, serverRpcClient) + if totalCap < testCap { + fmt.Println("Total capacity underrun") + close(stop) + wg.Wait() + return false + } + + processed := processedSince(start) + var avg uint64 + fmt.Printf("Processed") + for i, p := range processed { + fmt.Printf(" %d", p) + processed[i] = uint64(float64(p) / weights[i]) + avg += processed[i] + } + avg /= uint64(len(processed)) + + if avg >= 10000 { + var maxDev float64 + for _, p := range processed { + dev := float64(int64(p-avg)) / float64(avg) + fmt.Printf(" %7.4f", dev) + if dev < 0 { + dev = -dev + } + if dev > maxDev { + maxDev = dev + } + } + fmt.Printf(" max deviation: %f totalCap: %d\n", maxDev, totalCap) + if maxDev <= testTolerance { + fmt.Println("success") + break + } + } else { + fmt.Println() + } + time.Sleep(time.Millisecond * 200) + } + } + + close(stop) + wg.Wait() + + for i, count := range reqCount { + fmt.Println("client", i, "processed", count) + } + return true + }) { + fmt.Println("restarting test") + } +} + +func getHead(ctx context.Context, t *testing.T, client *rpc.Client) (uint64, common.Hash) { + res := make(map[string]interface{}) + if err := client.CallContext(ctx, &res, "eth_getBlockByNumber", "latest", false); err != nil { + t.Fatalf("Failed to obtain head block: %v", err) + } + numStr, ok := res["number"].(string) + if !ok { + t.Fatalf("RPC block number field invalid") + } + num, err := hexutil.DecodeUint64(numStr) + if err != nil { + t.Fatalf("Failed to decode RPC block number: %v", err) + } + hashStr, ok := res["hash"].(string) + if !ok { + t.Fatalf("RPC block number field invalid") + } + hash := common.HexToHash(hashStr) + return num, hash +} + +func testRequest(ctx context.Context, t *testing.T, client *rpc.Client) bool { + //res := make(map[string]interface{}) + var res string + var addr common.Address + rand.Read(addr[:]) + c, _ := context.WithTimeout(ctx, time.Second*12) + // if err := client.CallContext(ctx, &res, "eth_getProof", addr, nil, "latest"); err != nil { + err := client.CallContext(c, &res, "eth_getBalance", addr, "latest") + if err != nil { + fmt.Println("request error:", err) + } + return err == nil +} + +func setCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID, cap uint64) { + if err := server.CallContext(ctx, nil, "les_setClientCapacity", clientID, cap); err != nil { + t.Fatalf("Failed to set client capacity: %v", err) + } +} + +func getCapacity(ctx context.Context, t *testing.T, server *rpc.Client, clientID enode.ID) uint64 { + var s string + if err := server.CallContext(ctx, &s, "les_getClientCapacity", clientID); err != nil { + t.Fatalf("Failed to get client capacity: %v", err) + } + cap, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode client capacity: %v", err) + } + return cap +} + +func getTotalCap(ctx context.Context, t *testing.T, server *rpc.Client) uint64 { + var s string + if err := server.CallContext(ctx, &s, "les_totalCapacity"); err != nil { + t.Fatalf("Failed to query total capacity: %v", err) + } + total, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode total capacity: %v", err) + } + return total +} + +func getMinCap(ctx context.Context, t *testing.T, server *rpc.Client) uint64 { + var s string + if err := server.CallContext(ctx, &s, "les_minimumCapacity"); err != nil { + t.Fatalf("Failed to query minimum capacity: %v", err) + } + min, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode minimum capacity: %v", err) + } + return min +} + +func getFreeCap(ctx context.Context, t *testing.T, server *rpc.Client) uint64 { + var s string + if err := server.CallContext(ctx, &s, "les_freeClientCapacity"); err != nil { + t.Fatalf("Failed to query free client capacity: %v", err) + } + free, err := hexutil.DecodeUint64(s) + if err != nil { + t.Fatalf("Failed to decode free client capacity: %v", err) + } + return free +} + +func init() { + flag.Parse() + // register the Delivery service which will run as a devp2p + // protocol when using the exec adapter + adapters.RegisterServices(services) + + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) +} + +var ( + adapter = flag.String("adapter", "exec", "type of simulation: sim|socket|exec|docker") + loglevel = flag.Int("loglevel", 0, "verbosity of logs") + nodes = flag.Int("nodes", 0, "number of nodes") +) + +var services = adapters.Services{ + "lesclient": newLesClientService, + "lesserver": newLesServerService, +} + +func NewNetwork() (*simulations.Network, func(), error) { + adapter, adapterTeardown, err := NewAdapter(*adapter, services) + if err != nil { + return nil, adapterTeardown, err + } + defaultService := "streamer" + net := simulations.NewNetwork(adapter, &simulations.NetworkConfig{ + ID: "0", + DefaultService: defaultService, + }) + teardown := func() { + adapterTeardown() + net.Shutdown() + } + + return net, teardown, nil +} + +func NewAdapter(adapterType string, services adapters.Services) (adapter adapters.NodeAdapter, teardown func(), err error) { + teardown = func() {} + switch adapterType { + case "sim": + adapter = adapters.NewSimAdapter(services) + // case "socket": + // adapter = adapters.NewSocketAdapter(services) + case "exec": + baseDir, err0 := ioutil.TempDir("", "les-test") + if err0 != nil { + return nil, teardown, err0 + } + teardown = func() { os.RemoveAll(baseDir) } + adapter = adapters.NewExecAdapter(baseDir) + /*case "docker": + adapter, err = adapters.NewDockerAdapter() + if err != nil { + return nil, teardown, err + }*/ + default: + return nil, teardown, errors.New("adapter needs to be one of sim, socket, exec, docker") + } + return adapter, teardown, nil +} + +func testSim(t *testing.T, serverCount, clientCount int, serverDir, clientDir []string, test func(ctx context.Context, net *simulations.Network, servers []*simulations.Node, clients []*simulations.Node) bool) bool { + net, teardown, err := NewNetwork() + defer teardown() + if err != nil { + t.Fatalf("Failed to create network: %v", err) + } + timeout := 1800 * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + servers := make([]*simulations.Node, serverCount) + clients := make([]*simulations.Node, clientCount) + + for i := range clients { + clientconf := adapters.RandomNodeConfig() + clientconf.Services = []string{"lesclient"} + if len(clientDir) == clientCount { + clientconf.DataDir = clientDir[i] + } + client, err := net.NewNodeWithConfig(clientconf) + if err != nil { + t.Fatalf("Failed to create client: %v", err) + } + clients[i] = client + } + + for i := range servers { + serverconf := adapters.RandomNodeConfig() + serverconf.Services = []string{"lesserver"} + if len(serverDir) == serverCount { + serverconf.DataDir = serverDir[i] + } + server, err := net.NewNodeWithConfig(serverconf) + if err != nil { + t.Fatalf("Failed to create server: %v", err) + } + servers[i] = server + } + + for _, client := range clients { + if err := net.Start(client.ID()); err != nil { + t.Fatalf("Failed to start client node: %v", err) + } + } + for _, server := range servers { + if err := net.Start(server.ID()); err != nil { + t.Fatalf("Failed to start server node: %v", err) + } + } + + return test(ctx, net, servers, clients) +} + +func newLesClientService(ctx *adapters.ServiceContext) (node.Service, error) { + config := eth.DefaultConfig + config.SyncMode = downloader.LightSync + config.Ethash.PowMode = ethash.ModeFake + return New(ctx.NodeContext, &config) +} + +func newLesServerService(ctx *adapters.ServiceContext) (node.Service, error) { + config := eth.DefaultConfig + config.SyncMode = downloader.FullSync + config.LightServ = testServerCapacity + config.LightPeers = testMaxClients + ethereum, err := eth.New(ctx.NodeContext, &config) + if err != nil { + return nil, err + } + + server, err := NewLesServer(ethereum, &config) + if err != nil { + return nil, err + } + ethereum.AddLesServer(server) + return ethereum, nil +} diff --git a/les/backend.go b/les/backend.go index cd99f8f813..67ddf17e4c 100644 --- a/les/backend.go +++ b/les/backend.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" @@ -100,7 +101,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { chainConfig: chainConfig, eventMux: ctx.EventMux, peers: peers, - reqDist: newRequestDistributor(peers, quitSync), + reqDist: newRequestDistributor(peers, quitSync, &mclock.System{}), accountManager: ctx.AccountManager, engine: eth.CreateConsensusEngine(ctx, chainConfig, &config.Ethash, nil, false, chainDb), shutdownChan: make(chan bool), diff --git a/les/benchmark.go b/les/benchmark.go new file mode 100644 index 0000000000..cb302c6ea5 --- /dev/null +++ b/les/benchmark.go @@ -0,0 +1,353 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "encoding/binary" + "fmt" + "math/big" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" +) + +// requestBenchmark is an interface for different randomized request generators +type requestBenchmark interface { + // init initializes the generator for generating the given number of randomized requests + init(pm *ProtocolManager, count int) error + // request initiates sending a single request to the given peer + request(peer *peer, index int) error +} + +// benchmarkBlockHeaders implements requestBenchmark +type benchmarkBlockHeaders struct { + amount, skip int + reverse, byHash bool + offset, randMax int64 + hashes []common.Hash +} + +func (b *benchmarkBlockHeaders) init(pm *ProtocolManager, count int) error { + d := int64(b.amount-1) * int64(b.skip+1) + b.offset = 0 + b.randMax = pm.blockchain.CurrentHeader().Number.Int64() + 1 - d + if b.randMax < 0 { + return fmt.Errorf("chain is too short") + } + if b.reverse { + b.offset = d + } + if b.byHash { + b.hashes = make([]common.Hash, count) + for i := range b.hashes { + b.hashes[i] = rawdb.ReadCanonicalHash(pm.chainDb, uint64(b.offset+rand.Int63n(b.randMax))) + } + } + return nil +} + +func (b *benchmarkBlockHeaders) request(peer *peer, index int) error { + if b.byHash { + return peer.RequestHeadersByHash(0, 0, b.hashes[index], b.amount, b.skip, b.reverse) + } else { + return peer.RequestHeadersByNumber(0, 0, uint64(b.offset+rand.Int63n(b.randMax)), b.amount, b.skip, b.reverse) + } +} + +// benchmarkBodiesOrReceipts implements requestBenchmark +type benchmarkBodiesOrReceipts struct { + receipts bool + hashes []common.Hash +} + +func (b *benchmarkBodiesOrReceipts) init(pm *ProtocolManager, count int) error { + randMax := pm.blockchain.CurrentHeader().Number.Int64() + 1 + b.hashes = make([]common.Hash, count) + for i := range b.hashes { + b.hashes[i] = rawdb.ReadCanonicalHash(pm.chainDb, uint64(rand.Int63n(randMax))) + } + return nil +} + +func (b *benchmarkBodiesOrReceipts) request(peer *peer, index int) error { + if b.receipts { + return peer.RequestReceipts(0, 0, []common.Hash{b.hashes[index]}) + } else { + return peer.RequestBodies(0, 0, []common.Hash{b.hashes[index]}) + } +} + +// benchmarkProofsOrCode implements requestBenchmark +type benchmarkProofsOrCode struct { + code bool + headHash common.Hash +} + +func (b *benchmarkProofsOrCode) init(pm *ProtocolManager, count int) error { + b.headHash = pm.blockchain.CurrentHeader().Hash() + return nil +} + +func (b *benchmarkProofsOrCode) request(peer *peer, index int) error { + key := make([]byte, 32) + rand.Read(key) + if b.code { + return peer.RequestCode(0, 0, []CodeReq{{BHash: b.headHash, AccKey: key}}) + } else { + return peer.RequestProofs(0, 0, []ProofReq{{BHash: b.headHash, Key: key}}) + } +} + +// benchmarkHelperTrie implements requestBenchmark +type benchmarkHelperTrie struct { + bloom bool + reqCount int + sectionCount, headNum uint64 +} + +func (b *benchmarkHelperTrie) init(pm *ProtocolManager, count int) error { + if b.bloom { + b.sectionCount, b.headNum, _ = pm.server.bloomTrieIndexer.Sections() + } else { + b.sectionCount, _, _ = pm.server.chtIndexer.Sections() + b.sectionCount /= (params.CHTFrequencyClient / params.CHTFrequencyServer) + b.headNum = b.sectionCount*params.CHTFrequencyClient - 1 + } + if b.sectionCount == 0 { + return fmt.Errorf("no processed sections available") + } + return nil +} + +func (b *benchmarkHelperTrie) request(peer *peer, index int) error { + reqs := make([]HelperTrieReq, b.reqCount) + + if b.bloom { + bitIdx := uint16(rand.Intn(2048)) + for i := range reqs { + key := make([]byte, 10) + binary.BigEndian.PutUint16(key[:2], bitIdx) + binary.BigEndian.PutUint64(key[2:], uint64(rand.Int63n(int64(b.sectionCount)))) + reqs[i] = HelperTrieReq{Type: htBloomBits, TrieIdx: b.sectionCount - 1, Key: key} + } + } else { + for i := range reqs { + key := make([]byte, 8) + binary.BigEndian.PutUint64(key[:], uint64(rand.Int63n(int64(b.headNum)))) + reqs[i] = HelperTrieReq{Type: htCanonical, TrieIdx: b.sectionCount - 1, Key: key, AuxReq: auxHeader} + } + } + + return peer.RequestHelperTrieProofs(0, 0, reqs) +} + +// benchmarkTxSend implements requestBenchmark +type benchmarkTxSend struct { + txs types.Transactions +} + +func (b *benchmarkTxSend) init(pm *ProtocolManager, count int) error { + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + signer := types.NewEIP155Signer(big.NewInt(18)) + b.txs = make(types.Transactions, count) + + for i := range b.txs { + data := make([]byte, txSizeCostLimit) + rand.Read(data) + tx, err := types.SignTx(types.NewTransaction(0, addr, new(big.Int), 0, new(big.Int), data), signer, key) + if err != nil { + panic(err) + } + b.txs[i] = tx + } + return nil +} + +func (b *benchmarkTxSend) request(peer *peer, index int) error { + enc, _ := rlp.EncodeToBytes(types.Transactions{b.txs[index]}) + return peer.SendTxs(0, 0, enc) +} + +// benchmarkTxStatus implements requestBenchmark +type benchmarkTxStatus struct{} + +func (b *benchmarkTxStatus) init(pm *ProtocolManager, count int) error { + return nil +} + +func (b *benchmarkTxStatus) request(peer *peer, index int) error { + var hash common.Hash + rand.Read(hash[:]) + return peer.RequestTxStatus(0, 0, []common.Hash{hash}) +} + +// benchmarkSetup stores measurement data for a single benchmark type +type benchmarkSetup struct { + req requestBenchmark + totalCount int + totalTime, avgTime time.Duration + maxInSize, maxOutSize uint32 + err error +} + +// runBenchmark runs a benchmark cycle for all benchmark types in the specified +// number of passes +func (pm *ProtocolManager) runBenchmark(benchmarks []requestBenchmark, passCount int, targetTime time.Duration) []*benchmarkSetup { + setup := make([]*benchmarkSetup, len(benchmarks)) + for i, b := range benchmarks { + setup[i] = &benchmarkSetup{req: b} + } + for i := 0; i < passCount; i++ { + log.Info("Running benchmark", "pass", i+1, "total", passCount) + todo := make([]*benchmarkSetup, len(benchmarks)) + copy(todo, setup) + for len(todo) > 0 { + // select a random element + index := rand.Intn(len(todo)) + next := todo[index] + todo[index] = todo[len(todo)-1] + todo = todo[:len(todo)-1] + + if next.err == nil { + // calculate request count + count := 50 + if next.totalTime > 0 { + count = int(uint64(next.totalCount) * uint64(targetTime) / uint64(next.totalTime)) + } + if err := pm.measure(next, count); err != nil { + next.err = err + } + } + } + } + log.Info("Benchmark completed") + + for _, s := range setup { + if s.err == nil { + s.avgTime = s.totalTime / time.Duration(s.totalCount) + } + } + return setup +} + +// meteredPipe implements p2p.MsgReadWriter and remembers the largest single +// message size sent through the pipe +type meteredPipe struct { + rw p2p.MsgReadWriter + maxSize uint32 +} + +func (m *meteredPipe) ReadMsg() (p2p.Msg, error) { + return m.rw.ReadMsg() +} + +func (m *meteredPipe) WriteMsg(msg p2p.Msg) error { + if msg.Size > m.maxSize { + m.maxSize = msg.Size + } + return m.rw.WriteMsg(msg) +} + +// measure runs a benchmark for a single type in a single pass, with the given +// number of requests +func (pm *ProtocolManager) measure(setup *benchmarkSetup, count int) error { + clientPipe, serverPipe := p2p.MsgPipe() + clientMeteredPipe := &meteredPipe{rw: clientPipe} + serverMeteredPipe := &meteredPipe{rw: serverPipe} + var id enode.ID + rand.Read(id[:]) + clientPeer := pm.newPeer(lpv2, NetworkId, p2p.NewPeer(id, "client", nil), clientMeteredPipe) + serverPeer := pm.newPeer(lpv2, NetworkId, p2p.NewPeer(id, "server", nil), serverMeteredPipe) + serverPeer.sendQueue = newExecQueue(count) + serverPeer.announceType = announceTypeNone + serverPeer.fcCosts = make(requestCostTable) + c := &requestCosts{} + for code := range requests { + serverPeer.fcCosts[code] = c + } + serverPeer.fcParams = flowcontrol.ServerParams{BufLimit: 1, MinRecharge: 1} + serverPeer.fcClient = flowcontrol.NewClientNode(pm.server.fcManager, serverPeer.fcParams) + defer serverPeer.fcClient.Disconnect() + + if err := setup.req.init(pm, count); err != nil { + return err + } + + errCh := make(chan error, 10) + start := mclock.Now() + + go func() { + for i := 0; i < count; i++ { + if err := setup.req.request(clientPeer, i); err != nil { + errCh <- err + return + } + } + }() + go func() { + for i := 0; i < count; i++ { + if err := pm.handleMsg(serverPeer); err != nil { + errCh <- err + return + } + } + }() + go func() { + for i := 0; i < count; i++ { + msg, err := clientPipe.ReadMsg() + if err != nil { + errCh <- err + return + } + var i interface{} + msg.Decode(&i) + } + // at this point we can be sure that the other two + // goroutines finished successfully too + close(errCh) + }() + select { + case err := <-errCh: + if err != nil { + return err + } + case <-pm.quitSync: + clientPipe.Close() + serverPipe.Close() + return fmt.Errorf("Benchmark cancelled") + } + + setup.totalTime += time.Duration(mclock.Now() - start) + setup.totalCount += count + setup.maxInSize = clientMeteredPipe.maxSize + setup.maxOutSize = serverMeteredPipe.maxSize + clientPipe.Close() + serverPipe.Close() + return nil +} diff --git a/les/costtracker.go b/les/costtracker.go new file mode 100644 index 0000000000..69531937ef --- /dev/null +++ b/les/costtracker.go @@ -0,0 +1,388 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more detailct. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "encoding/binary" + "math" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/les/flowcontrol" + "github.com/ethereum/go-ethereum/log" +) + +const makeCostStats = false // make request cost statistics during operation + +var ( + // average request cost estimates based on serving time + reqAvgTimeCost = requestCostTable{ + GetBlockHeadersMsg: {150000, 30000}, + GetBlockBodiesMsg: {0, 700000}, + GetReceiptsMsg: {0, 1000000}, + GetCodeMsg: {0, 450000}, + GetProofsV1Msg: {0, 600000}, + GetProofsV2Msg: {0, 600000}, + GetHeaderProofsMsg: {0, 1000000}, + GetHelperTrieProofsMsg: {0, 1000000}, + SendTxMsg: {0, 450000}, + SendTxV2Msg: {0, 450000}, + GetTxStatusMsg: {0, 250000}, + } + // maximum incoming message size estimates + reqMaxInSize = requestCostTable{ + GetBlockHeadersMsg: {40, 0}, + GetBlockBodiesMsg: {0, 40}, + GetReceiptsMsg: {0, 40}, + GetCodeMsg: {0, 80}, + GetProofsV1Msg: {0, 80}, + GetProofsV2Msg: {0, 80}, + GetHeaderProofsMsg: {0, 20}, + GetHelperTrieProofsMsg: {0, 20}, + SendTxMsg: {0, 66000}, + SendTxV2Msg: {0, 66000}, + GetTxStatusMsg: {0, 50}, + } + // maximum outgoing message size estimates + reqMaxOutSize = requestCostTable{ + GetBlockHeadersMsg: {0, 556}, + GetBlockBodiesMsg: {0, 100000}, + GetReceiptsMsg: {0, 200000}, + GetCodeMsg: {0, 50000}, + GetProofsV1Msg: {0, 4000}, + GetProofsV2Msg: {0, 4000}, + GetHeaderProofsMsg: {0, 4000}, + GetHelperTrieProofsMsg: {0, 4000}, + SendTxMsg: {0, 0}, + SendTxV2Msg: {0, 100}, + GetTxStatusMsg: {0, 100}, + } + minBufLimit = uint64(50000000 * maxCostFactor) // minimum buffer limit allowed for a client + minCapacity = (minBufLimit-1)/bufLimitRatio + 1 // minimum capacity allowed for a client +) + +const ( + maxCostFactor = 2 // ratio of maximum and average cost estimates + gfInitWeight = time.Second * 10 + gfMaxWeight = time.Hour + gfUsageThreshold = 0.5 + gfUsageTC = time.Second + gfDbKey = "_globalCostFactor" +) + +// costTracker is responsible for calculating costs and cost estimates on the +// server side. It continuously updates the global cost factor which is defined +// as the number of cost units per nanosecond of serving time in a single thread. +// It is based on statistics collected during serving requests in high-load periods +// and practically acts as a one-dimension request price scaling factor over the +// pre-defined cost estimate table. Instead of scaling the cost values, the real +// value of cost units is changed by applying the factor to the serving times. This +// is more convenient because the changes in the cost factor can be applied immediately +// without always notifying the clients about the changed cost tables. +type costTracker struct { + db ethdb.Database + stopCh chan chan struct{} + + inSizeFactor, outSizeFactor float64 + gf, utilTarget float64 + + gfUpdateCh chan gfUpdate + gfLock sync.RWMutex + totalRechargeCh chan uint64 + + stats map[uint64][]uint64 +} + +// newCostTracker creates a cost tracker and loads the cost factor statistics from the database +func newCostTracker(db ethdb.Database, config *eth.Config) *costTracker { + utilTarget := float64(config.LightServ) * flowcontrol.FixedPointMultiplier / 100 + ct := &costTracker{ + db: db, + stopCh: make(chan chan struct{}), + utilTarget: utilTarget, + } + if config.LightBandwidthIn > 0 { + ct.inSizeFactor = utilTarget / float64(config.LightBandwidthIn) + } + if config.LightBandwidthOut > 0 { + ct.outSizeFactor = utilTarget / float64(config.LightBandwidthOut) + } + if makeCostStats { + ct.stats = make(map[uint64][]uint64) + for code := range reqAvgTimeCost { + ct.stats[code] = make([]uint64, 10) + } + } + ct.gfLoop() + return ct +} + +// stop stops the cost tracker and saves the cost factor statistics to the database +func (ct *costTracker) stop() { + stopCh := make(chan struct{}) + ct.stopCh <- stopCh + <-stopCh + if makeCostStats { + ct.printStats() + } +} + +// makeCostList returns upper cost estimates based on the hardcoded cost estimate +// tables and the optionally specified incoming/outgoing bandwidth limits +func (ct *costTracker) makeCostList() RequestCostList { + maxCost := func(avgTime, inSize, outSize uint64) uint64 { + globalFactor := ct.globalFactor() + + cost := avgTime * maxCostFactor + inSizeCost := uint64(float64(inSize) * ct.inSizeFactor * globalFactor * maxCostFactor) + if inSizeCost > cost { + cost = inSizeCost + } + outSizeCost := uint64(float64(outSize) * ct.outSizeFactor * globalFactor * maxCostFactor) + if outSizeCost > cost { + cost = outSizeCost + } + return cost + } + var list RequestCostList + for code, data := range reqAvgTimeCost { + list = append(list, requestCostListItem{ + MsgCode: code, + BaseCost: maxCost(data.baseCost, reqMaxInSize[code].baseCost, reqMaxOutSize[code].baseCost), + ReqCost: maxCost(data.reqCost, reqMaxInSize[code].reqCost, reqMaxOutSize[code].reqCost), + }) + } + return list +} + +type gfUpdate struct { + avgTime, servingTime float64 +} + +// gfLoop starts an event loop which updates the global cost factor which is +// calculated as a weighted average of the average estimate / serving time ratio. +// The applied weight equals the serving time if gfUsage is over a threshold, +// zero otherwise. gfUsage is the recent average serving time per time unit in +// an exponential moving window. This ensures that statistics are collected only +// under high-load circumstances where the measured serving times are relevant. +// The total recharge parameter of the flow control system which controls the +// total allowed serving time per second but nominated in cost units, should +// also be scaled with the cost factor and is also updated by this loop. +func (ct *costTracker) gfLoop() { + var gfUsage, gfSum, gfWeight float64 + lastUpdate := mclock.Now() + expUpdate := lastUpdate + + data, _ := ct.db.Get([]byte(gfDbKey)) + if len(data) == 16 { + gfSum = math.Float64frombits(binary.BigEndian.Uint64(data[0:8])) + gfWeight = math.Float64frombits(binary.BigEndian.Uint64(data[8:16])) + } + if gfWeight < float64(gfInitWeight) { + gfSum = float64(gfInitWeight) + gfWeight = float64(gfInitWeight) + } + gf := gfSum / gfWeight + ct.gf = gf + ct.gfUpdateCh = make(chan gfUpdate, 100) + + go func() { + for { + select { + case r := <-ct.gfUpdateCh: + now := mclock.Now() + max := r.servingTime * gf + if r.avgTime > max { + max = r.avgTime + } + dt := float64(now - expUpdate) + expUpdate = now + gfUsage = gfUsage*math.Exp(-dt/float64(gfUsageTC)) + max*1000000/float64(gfUsageTC) + + if gfUsage >= gfUsageThreshold*ct.utilTarget*gf { + gfSum += r.avgTime + gfWeight += r.servingTime + if time.Duration(now-lastUpdate) > time.Second { + gf = gfSum / gfWeight + if gfWeight >= float64(gfMaxWeight) { + gfSum = gf * float64(gfMaxWeight) + gfWeight = float64(gfMaxWeight) + } + lastUpdate = now + ct.gfLock.Lock() + ct.gf = gf + ch := ct.totalRechargeCh + ct.gfLock.Unlock() + if ch != nil { + select { + case ct.totalRechargeCh <- uint64(ct.utilTarget * gf): + default: + } + } + log.Debug("global cost factor updated", "gf", gf, "weight", time.Duration(gfWeight)) + } + } + case stopCh := <-ct.stopCh: + var data [16]byte + binary.BigEndian.PutUint64(data[0:8], math.Float64bits(gfSum)) + binary.BigEndian.PutUint64(data[8:16], math.Float64bits(gfWeight)) + ct.db.Put([]byte(gfDbKey), data[:]) + log.Debug("global cost factor saved", "sum", time.Duration(gfSum), "weight", time.Duration(gfWeight)) + close(stopCh) + return + } + } + }() +} + +// globalFactor returns the current value of the global cost factor +func (ct *costTracker) globalFactor() float64 { + ct.gfLock.RLock() + defer ct.gfLock.RUnlock() + + return ct.gf +} + +// totalRecharge returns the current total recharge parameter which is used by +// flowcontrol.ClientManager and is scaled by the global cost factor +func (ct *costTracker) totalRecharge() uint64 { + ct.gfLock.RLock() + defer ct.gfLock.RUnlock() + + return uint64(ct.gf * ct.utilTarget) +} + +// subscribeTotalRecharge returns all future updates to the total recharge value +// through a channel and also returns the current value +func (ct *costTracker) subscribeTotalRecharge(ch chan uint64) uint64 { + ct.gfLock.Lock() + defer ct.gfLock.Unlock() + + ct.totalRechargeCh = ch + return uint64(ct.gf * ct.utilTarget) +} + +// updateStats updates the global cost factor and (if enabled) the real cost vs. +// average estimate statistics +func (ct *costTracker) updateStats(code, amount, servingTime, realCost uint64) { + avg := reqAvgTimeCost[code] + avgTime := avg.baseCost + amount*avg.reqCost + select { + case ct.gfUpdateCh <- gfUpdate{float64(avgTime), float64(servingTime)}: + default: + } + if makeCostStats { + realCost <<= 4 + l := 0 + for l < 9 && realCost > avgTime { + l++ + realCost >>= 1 + } + atomic.AddUint64(&ct.stats[code][l], 1) + } +} + +// realCost calculates the final cost of a request based on actual serving time, +// incoming and outgoing message size +// +// Note: message size is only taken into account if bandwidth limitation is applied +// and the cost based on either message size is greater than the cost based on +// serving time. A maximum of the three costs is applied instead of their sum +// because the three limited resources (serving thread time and i/o bandwidth) can +// also be maxed out simultaneously. +func (ct *costTracker) realCost(servingTime uint64, inSize, outSize uint32) uint64 { + cost := float64(servingTime) + inSizeCost := float64(inSize) * ct.inSizeFactor + if inSizeCost > cost { + cost = inSizeCost + } + outSizeCost := float64(outSize) * ct.outSizeFactor + if outSizeCost > cost { + cost = outSizeCost + } + return uint64(cost * ct.globalFactor()) +} + +// printStats prints the distribution of real request cost relative to the average estimates +func (ct *costTracker) printStats() { + if ct.stats == nil { + return + } + for code, arr := range ct.stats { + log.Info("Request cost statistics", "code", code, "1/16", arr[0], "1/8", arr[1], "1/4", arr[2], "1/2", arr[3], "1", arr[4], "2", arr[5], "4", arr[6], "8", arr[7], "16", arr[8], ">16", arr[9]) + } +} + +type ( + // requestCostTable assigns a cost estimate function to each request type + // which is a linear function of the requested amount + // (cost = baseCost + reqCost * amount) + requestCostTable map[uint64]*requestCosts + requestCosts struct { + baseCost, reqCost uint64 + } + + // RequestCostList is a list representation of request costs which is used for + // database storage and communication through the network + RequestCostList []requestCostListItem + requestCostListItem struct { + MsgCode, BaseCost, ReqCost uint64 + } +) + +// getCost calculates the estimated cost for a given request type and amount +func (table requestCostTable) getCost(code, amount uint64) uint64 { + costs := table[code] + return costs.baseCost + amount*costs.reqCost +} + +// decode converts a cost list to a cost table +func (list RequestCostList) decode() requestCostTable { + table := make(requestCostTable) + for _, e := range list { + table[e.MsgCode] = &requestCosts{ + baseCost: e.BaseCost, + reqCost: e.ReqCost, + } + } + return table +} + +// testCostList returns a dummy request cost list used by tests +func testCostList() RequestCostList { + cl := make(RequestCostList, len(reqAvgTimeCost)) + var max uint64 + for code := range reqAvgTimeCost { + if code > max { + max = code + } + } + i := 0 + for code := uint64(0); code <= max; code++ { + if _, ok := reqAvgTimeCost[code]; ok { + cl[i].MsgCode = code + cl[i].BaseCost = 0 + cl[i].ReqCost = 0 + i++ + } + } + return cl +} diff --git a/les/distributor.go b/les/distributor.go index f90765b624..1de267f27f 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -14,20 +14,21 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package les import ( "container/list" "sync" "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) // requestDistributor implements a mechanism that distributes requests to // suitable peers, obeying flow control rules and prioritizing them in creation // order (even when a resend is necessary). type requestDistributor struct { + clock mclock.Clock reqQueue *list.List lastReqOrder uint64 peers map[distPeer]struct{} @@ -67,8 +68,9 @@ type distReq struct { } // newRequestDistributor creates a new request distributor -func newRequestDistributor(peers *peerSet, stopChn chan struct{}) *requestDistributor { +func newRequestDistributor(peers *peerSet, stopChn chan struct{}, clock mclock.Clock) *requestDistributor { d := &requestDistributor{ + clock: clock, reqQueue: list.New(), loopChn: make(chan struct{}, 2), stopChn: stopChn, @@ -148,7 +150,7 @@ func (d *requestDistributor) loop() { wait = distMaxWait } go func() { - time.Sleep(wait) + d.clock.Sleep(wait) d.loopChn <- struct{}{} }() break loop diff --git a/les/distributor_test.go b/les/distributor_test.go index 8c7621f26b..d2247212cd 100644 --- a/les/distributor_test.go +++ b/les/distributor_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package les import ( @@ -23,6 +21,8 @@ import ( "sync" "testing" "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) type testDistReq struct { @@ -121,7 +121,7 @@ func testRequestDistributor(t *testing.T, resend bool) { stop := make(chan struct{}) defer close(stop) - dist := newRequestDistributor(nil, stop) + dist := newRequestDistributor(nil, stop, &mclock.System{}) var peers [testDistPeerCount]*testDistPeer for i := range peers { peers[i] = &testDistPeer{} diff --git a/les/fetcher.go b/les/fetcher.go index aa3101af70..057552f532 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( @@ -559,7 +558,7 @@ func (f *lightFetcher) newFetcherDistReq(bestHash common.Hash, reqID uint64, bes f.lock.Unlock() cost := p.GetRequestCost(GetBlockHeadersMsg, int(bestAmount)) - p.fcServer.QueueRequest(reqID, cost) + p.fcServer.QueuedRequest(reqID, cost) f.reqMu.Lock() f.requested[reqID] = fetchRequest{hash: bestHash, amount: bestAmount, peer: p, sent: mclock.Now()} f.reqMu.Unlock() diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 8ef4ba511f..c03f673b28 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -18,166 +18,339 @@ package flowcontrol import ( + "fmt" "sync" "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" ) -const fcTimeConst = time.Millisecond +const ( + // fcTimeConst is the time constant applied for MinRecharge during linear + // buffer recharge period + fcTimeConst = time.Millisecond + // DecParamDelay is applied at server side when decreasing capacity in order to + // avoid a buffer underrun error due to requests sent by the client before + // receiving the capacity update announcement + DecParamDelay = time.Second * 2 + // keepLogs is the duration of keeping logs; logging is not used if zero + keepLogs = 0 +) +// ServerParams are the flow control parameters specified by a server for a client +// +// Note: a server can assign different amounts of capacity to each client by giving +// different parameters to them. type ServerParams struct { BufLimit, MinRecharge uint64 } -type ClientNode struct { - params *ServerParams - bufValue uint64 - lastTime mclock.AbsTime - lock sync.Mutex - cm *ClientManager - cmNode *cmNode +// scheduledUpdate represents a delayed flow control parameter update +type scheduledUpdate struct { + time mclock.AbsTime + params ServerParams } -func NewClientNode(cm *ClientManager, params *ServerParams) *ClientNode { +// ClientNode is the flow control system's representation of a client +// (used in server mode only) +type ClientNode struct { + params ServerParams + bufValue uint64 + lastTime mclock.AbsTime + updateSchedule []scheduledUpdate + sumCost uint64 // sum of req costs received from this client + accepted map[uint64]uint64 // value = sumCost after accepting the given req + lock sync.Mutex + cm *ClientManager + log *logger + cmNodeFields +} + +// NewClientNode returns a new ClientNode +func NewClientNode(cm *ClientManager, params ServerParams) *ClientNode { node := &ClientNode{ cm: cm, params: params, bufValue: params.BufLimit, - lastTime: mclock.Now(), + lastTime: cm.clock.Now(), + accepted: make(map[uint64]uint64), } - node.cmNode = cm.addNode(node) + if keepLogs > 0 { + node.log = newLogger(keepLogs) + } + cm.connect(node) return node } -func (peer *ClientNode) Remove(cm *ClientManager) { - cm.removeNode(peer.cmNode) +// Disconnect should be called when a client is disconnected +func (node *ClientNode) Disconnect() { + node.cm.disconnect(node) } -func (peer *ClientNode) recalcBV(time mclock.AbsTime) { - dt := uint64(time - peer.lastTime) - if time < peer.lastTime { +// update recalculates the buffer value at a specified time while also performing +// scheduled flow control parameter updates if necessary +func (node *ClientNode) update(now mclock.AbsTime) { + for len(node.updateSchedule) > 0 && node.updateSchedule[0].time <= now { + node.recalcBV(node.updateSchedule[0].time) + node.updateParams(node.updateSchedule[0].params, now) + node.updateSchedule = node.updateSchedule[1:] + } + node.recalcBV(now) +} + +// recalcBV recalculates the buffer value at a specified time +func (node *ClientNode) recalcBV(now mclock.AbsTime) { + dt := uint64(now - node.lastTime) + if now < node.lastTime { dt = 0 } - peer.bufValue += peer.params.MinRecharge * dt / uint64(fcTimeConst) - if peer.bufValue > peer.params.BufLimit { - peer.bufValue = peer.params.BufLimit + node.bufValue += node.params.MinRecharge * dt / uint64(fcTimeConst) + if node.bufValue > node.params.BufLimit { + node.bufValue = node.params.BufLimit } - peer.lastTime = time + if node.log != nil { + node.log.add(now, fmt.Sprintf("updated bv=%d MRR=%d BufLimit=%d", node.bufValue, node.params.MinRecharge, node.params.BufLimit)) + } + node.lastTime = now } -func (peer *ClientNode) AcceptRequest() (uint64, bool) { - peer.lock.Lock() - defer peer.lock.Unlock() +// UpdateParams updates the flow control parameters of a client node +func (node *ClientNode) UpdateParams(params ServerParams) { + node.lock.Lock() + defer node.lock.Unlock() - time := mclock.Now() - peer.recalcBV(time) - return peer.bufValue, peer.cm.accept(peer.cmNode, time) -} - -func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) { - peer.lock.Lock() - defer peer.lock.Unlock() - - time := mclock.Now() - peer.recalcBV(time) - peer.bufValue -= cost - rcValue, rcost := peer.cm.processed(peer.cmNode, time) - if rcValue < peer.params.BufLimit { - bv := peer.params.BufLimit - rcValue - if bv > peer.bufValue { - peer.bufValue = bv + now := node.cm.clock.Now() + node.update(now) + if params.MinRecharge >= node.params.MinRecharge { + node.updateSchedule = nil + node.updateParams(params, now) + } else { + for i, s := range node.updateSchedule { + if params.MinRecharge >= s.params.MinRecharge { + s.params = params + node.updateSchedule = node.updateSchedule[:i+1] + return + } } + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) } - return peer.bufValue, rcost } +// updateParams updates the flow control parameters of the node +func (node *ClientNode) updateParams(params ServerParams, now mclock.AbsTime) { + diff := params.BufLimit - node.params.BufLimit + if int64(diff) > 0 { + node.bufValue += diff + } else if node.bufValue > params.BufLimit { + node.bufValue = params.BufLimit + } + node.cm.updateParams(node, params, now) +} + +// AcceptRequest returns whether a new request can be accepted and the missing +// buffer amount if it was rejected due to a buffer underrun. If accepted, maxCost +// is deducted from the flow control buffer. +func (node *ClientNode) AcceptRequest(reqID, index, maxCost uint64) (accepted bool, bufShort uint64, priority int64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.cm.clock.Now() + node.update(now) + if maxCost > node.bufValue { + if node.log != nil { + node.log.add(now, fmt.Sprintf("rejected reqID=%d bv=%d maxCost=%d", reqID, node.bufValue, maxCost)) + node.log.dump(now) + } + return false, maxCost - node.bufValue, 0 + } + node.bufValue -= maxCost + node.sumCost += maxCost + if node.log != nil { + node.log.add(now, fmt.Sprintf("accepted reqID=%d bv=%d maxCost=%d sumCost=%d", reqID, node.bufValue, maxCost, node.sumCost)) + } + node.accepted[index] = node.sumCost + return true, 0, node.cm.accepted(node, maxCost, now) +} + +// RequestProcessed should be called when the request has been processed +func (node *ClientNode) RequestProcessed(reqID, index, maxCost, realCost uint64) (bv uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.cm.clock.Now() + node.update(now) + node.cm.processed(node, maxCost, realCost, now) + bv = node.bufValue + node.sumCost - node.accepted[index] + if node.log != nil { + node.log.add(now, fmt.Sprintf("processed reqID=%d bv=%d maxCost=%d realCost=%d sumCost=%d oldSumCost=%d reportedBV=%d", reqID, node.bufValue, maxCost, realCost, node.sumCost, node.accepted[index], bv)) + } + delete(node.accepted, index) + return +} + +// ServerNode is the flow control system's representation of a server +// (used in client mode only) type ServerNode struct { + clock mclock.Clock bufEstimate uint64 + bufRecharge bool lastTime mclock.AbsTime - params *ServerParams + params ServerParams sumCost uint64 // sum of req costs sent to this server pending map[uint64]uint64 // value = sumCost after sending the given req + log *logger lock sync.RWMutex } -func NewServerNode(params *ServerParams) *ServerNode { - return &ServerNode{ +// NewServerNode returns a new ServerNode +func NewServerNode(params ServerParams, clock mclock.Clock) *ServerNode { + node := &ServerNode{ + clock: clock, bufEstimate: params.BufLimit, - lastTime: mclock.Now(), + bufRecharge: false, + lastTime: clock.Now(), params: params, pending: make(map[uint64]uint64), } + if keepLogs > 0 { + node.log = newLogger(keepLogs) + } + return node } -func (peer *ServerNode) recalcBLE(time mclock.AbsTime) { - dt := uint64(time - peer.lastTime) - if time < peer.lastTime { - dt = 0 +// UpdateParams updates the flow control parameters of the node +func (node *ServerNode) UpdateParams(params ServerParams) { + node.lock.Lock() + defer node.lock.Unlock() + + node.recalcBLE(mclock.Now()) + if params.BufLimit > node.params.BufLimit { + node.bufEstimate += params.BufLimit - node.params.BufLimit + } else { + if node.bufEstimate > params.BufLimit { + node.bufEstimate = params.BufLimit + } } - peer.bufEstimate += peer.params.MinRecharge * dt / uint64(fcTimeConst) - if peer.bufEstimate > peer.params.BufLimit { - peer.bufEstimate = peer.params.BufLimit + node.params = params +} + +// recalcBLE recalculates the lowest estimate for the client's buffer value at +// the given server at the specified time +func (node *ServerNode) recalcBLE(now mclock.AbsTime) { + if now < node.lastTime { + return + } + if node.bufRecharge { + dt := uint64(now - node.lastTime) + node.bufEstimate += node.params.MinRecharge * dt / uint64(fcTimeConst) + if node.bufEstimate >= node.params.BufLimit { + node.bufEstimate = node.params.BufLimit + node.bufRecharge = false + } + } + node.lastTime = now + if node.log != nil { + node.log.add(now, fmt.Sprintf("updated bufEst=%d MRR=%d BufLimit=%d", node.bufEstimate, node.params.MinRecharge, node.params.BufLimit)) } - peer.lastTime = time } // safetyMargin is added to the flow control waiting time when estimated buffer value is low const safetyMargin = time.Millisecond -func (peer *ServerNode) canSend(maxCost uint64) (time.Duration, float64) { - peer.recalcBLE(mclock.Now()) - maxCost += uint64(safetyMargin) * peer.params.MinRecharge / uint64(fcTimeConst) - if maxCost > peer.params.BufLimit { - maxCost = peer.params.BufLimit - } - if peer.bufEstimate >= maxCost { - return 0, float64(peer.bufEstimate-maxCost) / float64(peer.params.BufLimit) - } - return time.Duration((maxCost - peer.bufEstimate) * uint64(fcTimeConst) / peer.params.MinRecharge), 0 -} - // CanSend returns the minimum waiting time required before sending a request // with the given maximum estimated cost. Second return value is the relative // estimated buffer level after sending the request (divided by BufLimit). -func (peer *ServerNode) CanSend(maxCost uint64) (time.Duration, float64) { - peer.lock.RLock() - defer peer.lock.RUnlock() +func (node *ServerNode) CanSend(maxCost uint64) (time.Duration, float64) { + node.lock.RLock() + defer node.lock.RUnlock() - return peer.canSend(maxCost) -} - -// QueueRequest should be called when the request has been assigned to the given -// server node, before putting it in the send queue. It is mandatory that requests -// are sent in the same order as the QueueRequest calls are made. -func (peer *ServerNode) QueueRequest(reqID, maxCost uint64) { - peer.lock.Lock() - defer peer.lock.Unlock() - - peer.bufEstimate -= maxCost - peer.sumCost += maxCost - peer.pending[reqID] = peer.sumCost -} - -// GotReply adjusts estimated buffer value according to the value included in -// the latest request reply. -func (peer *ServerNode) GotReply(reqID, bv uint64) { - - peer.lock.Lock() - defer peer.lock.Unlock() - - if bv > peer.params.BufLimit { - bv = peer.params.BufLimit + now := node.clock.Now() + node.recalcBLE(now) + maxCost += uint64(safetyMargin) * node.params.MinRecharge / uint64(fcTimeConst) + if maxCost > node.params.BufLimit { + maxCost = node.params.BufLimit } - sc, ok := peer.pending[reqID] + if node.bufEstimate >= maxCost { + relBuf := float64(node.bufEstimate-maxCost) / float64(node.params.BufLimit) + if node.log != nil { + node.log.add(now, fmt.Sprintf("canSend bufEst=%d maxCost=%d true relBuf=%f", node.bufEstimate, maxCost, relBuf)) + } + return 0, relBuf + } + timeLeft := time.Duration((maxCost - node.bufEstimate) * uint64(fcTimeConst) / node.params.MinRecharge) + if node.log != nil { + node.log.add(now, fmt.Sprintf("canSend bufEst=%d maxCost=%d false timeLeft=%v", node.bufEstimate, maxCost, timeLeft)) + } + return timeLeft, 0 +} + +// QueuedRequest should be called when the request has been assigned to the given +// server node, before putting it in the send queue. It is mandatory that requests +// are sent in the same order as the QueuedRequest calls are made. +func (node *ServerNode) QueuedRequest(reqID, maxCost uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.clock.Now() + node.recalcBLE(now) + // Note: we do not know when requests actually arrive to the server so bufRecharge + // is not turned on here if buffer was full; in this case it is going to be turned + // on by the first reply's bufValue feedback + if node.bufEstimate >= maxCost { + node.bufEstimate -= maxCost + } else { + log.Error("Queued request with insufficient buffer estimate") + node.bufEstimate = 0 + } + node.sumCost += maxCost + node.pending[reqID] = node.sumCost + if node.log != nil { + node.log.add(now, fmt.Sprintf("queued reqID=%d bufEst=%d maxCost=%d sumCost=%d", reqID, node.bufEstimate, maxCost, node.sumCost)) + } +} + +// ReceivedReply adjusts estimated buffer value according to the value included in +// the latest request reply. +func (node *ServerNode) ReceivedReply(reqID, bv uint64) { + node.lock.Lock() + defer node.lock.Unlock() + + now := node.clock.Now() + node.recalcBLE(now) + if bv > node.params.BufLimit { + bv = node.params.BufLimit + } + sc, ok := node.pending[reqID] if !ok { return } - delete(peer.pending, reqID) - cc := peer.sumCost - sc - peer.bufEstimate = 0 + delete(node.pending, reqID) + cc := node.sumCost - sc + newEstimate := uint64(0) if bv > cc { - peer.bufEstimate = bv - cc + newEstimate = bv - cc + } + if newEstimate > node.bufEstimate { + // Note: we never reduce the buffer estimate based on the reported value because + // this can only happen because of the delayed delivery of the latest reply. + // The lowest estimate based on the previous reply can still be considered valid. + node.bufEstimate = newEstimate + } + + node.bufRecharge = node.bufEstimate < node.params.BufLimit + node.lastTime = now + if node.log != nil { + node.log.add(now, fmt.Sprintf("received reqID=%d bufEst=%d reportedBv=%d sumCost=%d oldSumCost=%d", reqID, node.bufEstimate, bv, node.sumCost, sc)) + } +} + +// DumpLogs dumps the event log if logging is used +func (node *ServerNode) DumpLogs() { + node.lock.Lock() + defer node.lock.Unlock() + + if node.log != nil { + node.log.dump(node.clock.Now()) } - peer.lastTime = mclock.Now() } diff --git a/les/flowcontrol/logger.go b/les/flowcontrol/logger.go new file mode 100644 index 0000000000..fcd1285a59 --- /dev/null +++ b/les/flowcontrol/logger.go @@ -0,0 +1,65 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flowcontrol + +import ( + "fmt" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +// logger collects events in string format and discards events older than the +// "keep" parameter +type logger struct { + events map[uint64]logEvent + writePtr, delPtr uint64 + keep time.Duration +} + +// logEvent describes a single event +type logEvent struct { + time mclock.AbsTime + event string +} + +// newLogger creates a new logger +func newLogger(keep time.Duration) *logger { + return &logger{ + events: make(map[uint64]logEvent), + keep: keep, + } +} + +// add adds a new event and discards old events if possible +func (l *logger) add(now mclock.AbsTime, event string) { + keepAfter := now - mclock.AbsTime(l.keep) + for l.delPtr < l.writePtr && l.events[l.delPtr].time <= keepAfter { + delete(l.events, l.delPtr) + l.delPtr++ + } + l.events[l.writePtr] = logEvent{now, event} + l.writePtr++ +} + +// dump prints all stored events +func (l *logger) dump(now mclock.AbsTime) { + for i := l.delPtr; i < l.writePtr; i++ { + e := l.events[i] + fmt.Println(time.Duration(e.time-now), e.event) + } +} diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index 28cc6f0fe7..532e6a4050 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,211 +14,388 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package flowcontrol implements a client side flow control mechanism package flowcontrol import ( + "fmt" + "math" "sync" "time" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" ) -const rcConst = 1000000 - -type cmNode struct { - node *ClientNode - lastUpdate mclock.AbsTime - serving, recharging bool - rcWeight uint64 - rcValue, rcDelta, startValue int64 - finishRecharge mclock.AbsTime +// cmNodeFields are ClientNode fields used by the client manager +// Note: these fields are locked by the client manager's mutex +type cmNodeFields struct { + corrBufValue int64 // buffer value adjusted with the extra recharge amount + rcLastIntValue int64 // past recharge integrator value when corrBufValue was last updated + rcFullIntValue int64 // future recharge integrator value when corrBufValue will reach maximum + queueIndex int // position in the recharge queue (-1 if not queued) } -func (node *cmNode) update(time mclock.AbsTime) { - dt := int64(time - node.lastUpdate) - node.rcValue += node.rcDelta * dt / rcConst - node.lastUpdate = time - if node.recharging && time >= node.finishRecharge { - node.recharging = false - node.rcDelta = 0 - node.rcValue = 0 - } -} +// FixedPointMultiplier is applied to the recharge integrator and the recharge curve. +// +// Note: fixed point arithmetic is required for the integrator because it is a +// constantly increasing value that can wrap around int64 limits (which behavior is +// also supported by the priority queue). A floating point value would gradually lose +// precision in this application. +// The recharge curve and all recharge values are encoded as fixed point because +// sumRecharge is frequently updated by adding or subtracting individual recharge +// values and perfect precision is required. +const FixedPointMultiplier = 1000000 -func (node *cmNode) set(serving bool, simReqCnt, sumWeight uint64) { - if node.serving && !serving { - node.recharging = true - sumWeight += node.rcWeight - } - node.serving = serving - if node.recharging && serving { - node.recharging = false - sumWeight -= node.rcWeight - } - - node.rcDelta = 0 - if serving { - node.rcDelta = int64(rcConst / simReqCnt) - } - if node.recharging { - node.rcDelta = -int64(node.node.cm.rcRecharge * node.rcWeight / sumWeight) - node.finishRecharge = node.lastUpdate + mclock.AbsTime(node.rcValue*rcConst/(-node.rcDelta)) - } -} +var ( + capFactorDropTC = 1 / float64(time.Second*10) // time constant for dropping the capacity factor + capFactorRaiseTC = 1 / float64(time.Hour) // time constant for raising the capacity factor + capFactorRaiseThreshold = 0.75 // connected / total capacity ratio threshold for raising the capacity factor +) +// ClientManager controls the capacity assigned to the clients of a server. +// Since ServerParams guarantee a safe lower estimate for processable requests +// even in case of all clients being active, ClientManager calculates a +// corrigated buffer value and usually allows a higher remaining buffer value +// to be returned with each reply. type ClientManager struct { - lock sync.Mutex - nodes map[*cmNode]struct{} - simReqCnt, sumWeight, rcSumValue uint64 - maxSimReq, maxRcSum uint64 - rcRecharge uint64 - resumeQueue chan chan bool - time mclock.AbsTime + clock mclock.Clock + lock sync.Mutex + enabledCh chan struct{} + + curve PieceWiseLinear + sumRecharge, totalRecharge, totalConnected uint64 + capLogFactor, totalCapacity float64 + capLastUpdate mclock.AbsTime + totalCapacityCh chan uint64 + + // recharge integrator is increasing in each moment with a rate of + // (totalRecharge / sumRecharge)*FixedPointMultiplier or 0 if sumRecharge==0 + rcLastUpdate mclock.AbsTime // last time the recharge integrator was updated + rcLastIntValue int64 // last updated value of the recharge integrator + // recharge queue is a priority queue with currently recharging client nodes + // as elements. The priority value is rcFullIntValue which allows to quickly + // determine which client will first finish recharge. + rcQueue *prque.Prque } -func NewClientManager(rcTarget, maxSimReq, maxRcSum uint64) *ClientManager { +// NewClientManager returns a new client manager. +// Client manager enhances flow control performance by allowing client buffers +// to recharge quicker than the minimum guaranteed recharge rate if possible. +// The sum of all minimum recharge rates (sumRecharge) is updated each time +// a clients starts or finishes buffer recharging. Then an adjusted total +// recharge rate is calculated using a piecewise linear recharge curve: +// +// totalRecharge = curve(sumRecharge) +// (totalRecharge >= sumRecharge is enforced) +// +// Then the "bonus" buffer recharge is distributed between currently recharging +// clients proportionally to their minimum recharge rates. +// +// Note: total recharge is proportional to the average number of parallel running +// serving threads. A recharge value of 1000000 corresponds to one thread in average. +// The maximum number of allowed serving threads should always be considerably +// higher than the targeted average number. +// +// Note 2: although it is possible to specify a curve allowing the total target +// recharge starting from zero sumRecharge, it makes sense to add a linear ramp +// starting from zero in order to not let a single low-priority client use up +// the entire server capacity and thus ensure quick availability for others at +// any moment. +func NewClientManager(curve PieceWiseLinear, clock mclock.Clock) *ClientManager { cm := &ClientManager{ - nodes: make(map[*cmNode]struct{}), - resumeQueue: make(chan chan bool), - rcRecharge: rcConst * rcConst / (100*rcConst/rcTarget - rcConst), - maxSimReq: maxSimReq, - maxRcSum: maxRcSum, + clock: clock, + rcQueue: prque.New(func(a interface{}, i int) { a.(*ClientNode).queueIndex = i }), + capLastUpdate: clock.Now(), + } + if curve != nil { + cm.SetRechargeCurve(curve) } - go cm.queueProc() return cm } -func (self *ClientManager) Stop() { - self.lock.Lock() - defer self.lock.Unlock() +// SetRechargeCurve updates the recharge curve +func (cm *ClientManager) SetRechargeCurve(curve PieceWiseLinear) { + cm.lock.Lock() + defer cm.lock.Unlock() - // signal any waiting accept routines to return false - self.nodes = make(map[*cmNode]struct{}) - close(self.resumeQueue) -} - -func (self *ClientManager) addNode(cnode *ClientNode) *cmNode { - time := mclock.Now() - node := &cmNode{ - node: cnode, - lastUpdate: time, - finishRecharge: time, - rcWeight: 1, + now := cm.clock.Now() + cm.updateRecharge(now) + cm.updateCapFactor(now, false) + cm.curve = curve + if len(curve) > 0 { + cm.totalRecharge = curve[len(curve)-1].Y + } else { + cm.totalRecharge = 0 } - self.lock.Lock() - defer self.lock.Unlock() - - self.nodes[node] = struct{}{} - self.update(mclock.Now()) - return node + cm.refreshCapacity() } -func (self *ClientManager) removeNode(node *cmNode) { - self.lock.Lock() - defer self.lock.Unlock() +// connect should be called when a client is connected, before passing it to any +// other ClientManager function +func (cm *ClientManager) connect(node *ClientNode) { + cm.lock.Lock() + defer cm.lock.Unlock() - time := mclock.Now() - self.stop(node, time) - delete(self.nodes, node) - self.update(time) + now := cm.clock.Now() + cm.updateRecharge(now) + node.corrBufValue = int64(node.params.BufLimit) + node.rcLastIntValue = cm.rcLastIntValue + node.queueIndex = -1 + cm.updateCapFactor(now, true) + cm.totalConnected += node.params.MinRecharge } -// recalc sumWeight -func (self *ClientManager) updateNodes(time mclock.AbsTime) (rce bool) { - var sumWeight, rcSum uint64 - for node := range self.nodes { - rc := node.recharging - node.update(time) - if rc && !node.recharging { - rce = true - } - if node.recharging { - sumWeight += node.rcWeight - } - rcSum += uint64(node.rcValue) +// disconnect should be called when a client is disconnected +func (cm *ClientManager) disconnect(node *ClientNode) { + cm.lock.Lock() + defer cm.lock.Unlock() + + now := cm.clock.Now() + cm.updateRecharge(cm.clock.Now()) + cm.updateCapFactor(now, true) + cm.totalConnected -= node.params.MinRecharge +} + +// accepted is called when a request with given maximum cost is accepted. +// It returns a priority indicator for the request which is used to determine placement +// in the serving queue. Older requests have higher priority by default. If the client +// is almost out of buffer, request priority is reduced. +func (cm *ClientManager) accepted(node *ClientNode, maxCost uint64, now mclock.AbsTime) (priority int64) { + cm.lock.Lock() + defer cm.lock.Unlock() + + cm.updateNodeRc(node, -int64(maxCost), &node.params, now) + rcTime := (node.params.BufLimit - uint64(node.corrBufValue)) * FixedPointMultiplier / node.params.MinRecharge + return -int64(now) - int64(rcTime) +} + +// processed updates the client buffer according to actual request cost after +// serving has been finished. +// +// Note: processed should always be called for all accepted requests +func (cm *ClientManager) processed(node *ClientNode, maxCost, realCost uint64, now mclock.AbsTime) { + cm.lock.Lock() + defer cm.lock.Unlock() + + if realCost > maxCost { + realCost = maxCost + } + cm.updateNodeRc(node, int64(maxCost-realCost), &node.params, now) + if uint64(node.corrBufValue) > node.bufValue { + if node.log != nil { + node.log.add(now, fmt.Sprintf("corrected bv=%d oldBv=%d", node.corrBufValue, node.bufValue)) + } + node.bufValue = uint64(node.corrBufValue) } - self.sumWeight = sumWeight - self.rcSumValue = rcSum - return } -func (self *ClientManager) update(time mclock.AbsTime) { - for { - firstTime := time - for node := range self.nodes { - if node.recharging && node.finishRecharge < firstTime { - firstTime = node.finishRecharge - } +// updateParams updates the flow control parameters of a client node +func (cm *ClientManager) updateParams(node *ClientNode, params ServerParams, now mclock.AbsTime) { + cm.lock.Lock() + defer cm.lock.Unlock() + + cm.updateRecharge(now) + cm.updateCapFactor(now, true) + cm.totalConnected += params.MinRecharge - node.params.MinRecharge + cm.updateNodeRc(node, 0, ¶ms, now) +} + +// updateRecharge updates the recharge integrator and checks the recharge queue +// for nodes with recently filled buffers +func (cm *ClientManager) updateRecharge(now mclock.AbsTime) { + lastUpdate := cm.rcLastUpdate + cm.rcLastUpdate = now + // updating is done in multiple steps if node buffers are filled and sumRecharge + // is decreased before the given target time + for cm.sumRecharge > 0 { + bonusRatio := cm.curve.ValueAt(cm.sumRecharge) / float64(cm.sumRecharge) + if bonusRatio < 1 { + bonusRatio = 1 } - if self.updateNodes(firstTime) { - for node := range self.nodes { - if node.recharging { - node.set(node.serving, self.simReqCnt, self.sumWeight) - } - } - } else { - self.time = time + dt := now - lastUpdate + // fetch the client that finishes first + rcqNode := cm.rcQueue.PopItem().(*ClientNode) // if sumRecharge > 0 then the queue cannot be empty + // check whether it has already finished + dtNext := mclock.AbsTime(float64(rcqNode.rcFullIntValue-cm.rcLastIntValue) / bonusRatio) + if dt < dtNext { + // not finished yet, put it back, update integrator according + // to current bonusRatio and return + cm.rcQueue.Push(rcqNode, -rcqNode.rcFullIntValue) + cm.rcLastIntValue += int64(bonusRatio * float64(dt)) return } + lastUpdate += dtNext + // finished recharging, update corrBufValue and sumRecharge if necessary and do next step + if rcqNode.corrBufValue < int64(rcqNode.params.BufLimit) { + rcqNode.corrBufValue = int64(rcqNode.params.BufLimit) + cm.updateCapFactor(lastUpdate, true) + cm.sumRecharge -= rcqNode.params.MinRecharge + } + cm.rcLastIntValue = rcqNode.rcFullIntValue } } -func (self *ClientManager) canStartReq() bool { - return self.simReqCnt < self.maxSimReq && self.rcSumValue < self.maxRcSum +// updateNodeRc updates a node's corrBufValue and adds an external correction value. +// It also adds or removes the rcQueue entry and updates ServerParams and sumRecharge if necessary. +func (cm *ClientManager) updateNodeRc(node *ClientNode, bvc int64, params *ServerParams, now mclock.AbsTime) { + cm.updateRecharge(now) + wasFull := true + if node.corrBufValue != int64(node.params.BufLimit) { + wasFull = false + node.corrBufValue += (cm.rcLastIntValue - node.rcLastIntValue) * int64(node.params.MinRecharge) / FixedPointMultiplier + if node.corrBufValue > int64(node.params.BufLimit) { + node.corrBufValue = int64(node.params.BufLimit) + } + node.rcLastIntValue = cm.rcLastIntValue + } + node.corrBufValue += bvc + if node.corrBufValue < 0 { + node.corrBufValue = 0 + } + diff := int64(params.BufLimit - node.params.BufLimit) + if diff > 0 { + node.corrBufValue += diff + } + isFull := false + if node.corrBufValue >= int64(params.BufLimit) { + node.corrBufValue = int64(params.BufLimit) + isFull = true + } + sumRecharge := cm.sumRecharge + if !wasFull { + sumRecharge -= node.params.MinRecharge + } + if params != &node.params { + node.params = *params + } + if !isFull { + sumRecharge += node.params.MinRecharge + if node.queueIndex != -1 { + cm.rcQueue.Remove(node.queueIndex) + } + node.rcLastIntValue = cm.rcLastIntValue + node.rcFullIntValue = cm.rcLastIntValue + (int64(node.params.BufLimit)-node.corrBufValue)*FixedPointMultiplier/int64(node.params.MinRecharge) + cm.rcQueue.Push(node, -node.rcFullIntValue) + } + if sumRecharge != cm.sumRecharge { + cm.updateCapFactor(now, true) + cm.sumRecharge = sumRecharge + } + } -func (self *ClientManager) queueProc() { - for rc := range self.resumeQueue { - for { - time.Sleep(time.Millisecond * 10) - self.lock.Lock() - self.update(mclock.Now()) - cs := self.canStartReq() - self.lock.Unlock() - if cs { - break +// updateCapFactor updates the total capacity factor. The capacity factor allows +// the total capacity of the system to go over the allowed total recharge value +// if the sum of momentarily recharging clients only exceeds the total recharge +// allowance in a very small fraction of time. +// The capacity factor is dropped quickly (with a small time constant) if sumRecharge +// exceeds totalRecharge. It is raised slowly (with a large time constant) if most +// of the total capacity is used by connected clients (totalConnected is larger than +// totalCapacity*capFactorRaiseThreshold) and sumRecharge stays under +// totalRecharge*totalConnected/totalCapacity. +func (cm *ClientManager) updateCapFactor(now mclock.AbsTime, refresh bool) { + if cm.totalRecharge == 0 { + return + } + dt := now - cm.capLastUpdate + cm.capLastUpdate = now + + var d float64 + if cm.sumRecharge > cm.totalRecharge { + d = (1 - float64(cm.sumRecharge)/float64(cm.totalRecharge)) * capFactorDropTC + } else { + totalConnected := float64(cm.totalConnected) + var connRatio float64 + if totalConnected < cm.totalCapacity { + connRatio = totalConnected / cm.totalCapacity + } else { + connRatio = 1 + } + if connRatio > capFactorRaiseThreshold { + sumRecharge := float64(cm.sumRecharge) + limit := float64(cm.totalRecharge) * connRatio + if sumRecharge < limit { + d = (1 - sumRecharge/limit) * (connRatio - capFactorRaiseThreshold) * (1 / (1 - capFactorRaiseThreshold)) * capFactorRaiseTC } } - close(rc) } -} - -func (self *ClientManager) accept(node *cmNode, time mclock.AbsTime) bool { - self.lock.Lock() - defer self.lock.Unlock() - - self.update(time) - if !self.canStartReq() { - resume := make(chan bool) - self.lock.Unlock() - self.resumeQueue <- resume - <-resume - self.lock.Lock() - if _, ok := self.nodes[node]; !ok { - return false // reject if node has been removed or manager has been stopped + if d != 0 { + cm.capLogFactor += d * float64(dt) + if cm.capLogFactor < 0 { + cm.capLogFactor = 0 + } + if refresh { + cm.refreshCapacity() } } - self.simReqCnt++ - node.set(true, self.simReqCnt, self.sumWeight) - node.startValue = node.rcValue - self.update(self.time) - return true } -func (self *ClientManager) stop(node *cmNode, time mclock.AbsTime) { - if node.serving { - self.update(time) - self.simReqCnt-- - node.set(false, self.simReqCnt, self.sumWeight) - self.update(time) +// refreshCapacity recalculates the total capacity value and sends an update to the subscription +// channel if the relative change of the value since the last update is more than 0.1 percent +func (cm *ClientManager) refreshCapacity() { + totalCapacity := float64(cm.totalRecharge) * math.Exp(cm.capLogFactor) + if totalCapacity >= cm.totalCapacity*0.999 && totalCapacity <= cm.totalCapacity*1.001 { + return + } + cm.totalCapacity = totalCapacity + if cm.totalCapacityCh != nil { + select { + case cm.totalCapacityCh <- uint64(cm.totalCapacity): + default: + } } } -func (self *ClientManager) processed(node *cmNode, time mclock.AbsTime) (rcValue, rcCost uint64) { - self.lock.Lock() - defer self.lock.Unlock() +// SubscribeTotalCapacity returns all future updates to the total capacity value +// through a channel and also returns the current value +func (cm *ClientManager) SubscribeTotalCapacity(ch chan uint64) uint64 { + cm.lock.Lock() + defer cm.lock.Unlock() - self.stop(node, time) - return uint64(node.rcValue), uint64(node.rcValue - node.startValue) + cm.totalCapacityCh = ch + return uint64(cm.totalCapacity) +} + +// PieceWiseLinear is used to describe recharge curves +type PieceWiseLinear []struct{ X, Y uint64 } + +// ValueAt returns the curve's value at a given point +func (pwl PieceWiseLinear) ValueAt(x uint64) float64 { + l := 0 + h := len(pwl) + if h == 0 { + return 0 + } + for h != l { + m := (l + h) / 2 + if x > pwl[m].X { + l = m + 1 + } else { + h = m + } + } + if l == 0 { + return float64(pwl[0].Y) + } + l-- + if h == len(pwl) { + return float64(pwl[l].Y) + } + dx := pwl[h].X - pwl[l].X + if dx < 1 { + return float64(pwl[l].Y) + } + return float64(pwl[l].Y) + float64(pwl[h].Y-pwl[l].Y)*float64(x-pwl[l].X)/float64(dx) +} + +// Valid returns true if the X coordinates of the curve points are non-strictly monotonic +func (pwl PieceWiseLinear) Valid() bool { + var lastX uint64 + for _, i := range pwl { + if i.X < lastX { + return false + } + lastX = i.X + } + return true } diff --git a/les/flowcontrol/manager_test.go b/les/flowcontrol/manager_test.go new file mode 100644 index 0000000000..4e7746d400 --- /dev/null +++ b/les/flowcontrol/manager_test.go @@ -0,0 +1,123 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package flowcontrol + +import ( + "math/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" +) + +type testNode struct { + node *ClientNode + bufLimit, capacity uint64 + waitUntil mclock.AbsTime + index, totalCost uint64 +} + +const ( + testMaxCost = 1000000 + testLength = 100000 +) + +// testConstantTotalCapacity simulates multiple request sender nodes and verifies +// whether the total amount of served requests matches the expected value based on +// the total capacity and the duration of the test. +// Some nodes are sending requests occasionally so that their buffer should regularly +// reach the maximum while other nodes (the "max capacity nodes") are sending at the +// maximum permitted rate. The max capacity nodes are changed multiple times during +// a single test. +func TestConstantTotalCapacity(t *testing.T) { + testConstantTotalCapacity(t, 10, 1, 0) + testConstantTotalCapacity(t, 10, 1, 1) + testConstantTotalCapacity(t, 30, 1, 0) + testConstantTotalCapacity(t, 30, 2, 3) + testConstantTotalCapacity(t, 100, 1, 0) + testConstantTotalCapacity(t, 100, 3, 5) + testConstantTotalCapacity(t, 100, 5, 10) +} + +func testConstantTotalCapacity(t *testing.T, nodeCount, maxCapacityNodes, randomSend int) { + clock := &mclock.Simulated{} + nodes := make([]*testNode, nodeCount) + var totalCapacity uint64 + for i := range nodes { + nodes[i] = &testNode{capacity: uint64(50000 + rand.Intn(100000))} + totalCapacity += nodes[i].capacity + } + m := NewClientManager(PieceWiseLinear{{0, totalCapacity}}, clock) + for _, n := range nodes { + n.bufLimit = n.capacity * 6000 //uint64(2000+rand.Intn(10000)) + n.node = NewClientNode(m, ServerParams{BufLimit: n.bufLimit, MinRecharge: n.capacity}) + } + maxNodes := make([]int, maxCapacityNodes) + for i := range maxNodes { + // we don't care if some indexes are selected multiple times + // in that case we have fewer max nodes + maxNodes[i] = rand.Intn(nodeCount) + } + + for i := 0; i < testLength; i++ { + now := clock.Now() + for _, idx := range maxNodes { + for nodes[idx].send(t, now) { + } + } + if rand.Intn(testLength) < maxCapacityNodes*3 { + maxNodes[rand.Intn(maxCapacityNodes)] = rand.Intn(nodeCount) + } + + sendCount := randomSend + for sendCount > 0 { + if nodes[rand.Intn(nodeCount)].send(t, now) { + sendCount-- + } + } + + clock.Run(time.Millisecond) + } + + var totalCost uint64 + for _, n := range nodes { + totalCost += n.totalCost + } + ratio := float64(totalCost) / float64(totalCapacity) / testLength + if ratio < 0.98 || ratio > 1.02 { + t.Errorf("totalCost/totalCapacity/testLength ratio incorrect (expected: 1, got: %f)", ratio) + } + +} + +func (n *testNode) send(t *testing.T, now mclock.AbsTime) bool { + if now < n.waitUntil { + return false + } + n.index++ + if ok, _, _ := n.node.AcceptRequest(0, n.index, testMaxCost); !ok { + t.Fatalf("Rejected request after expected waiting time has passed") + } + rcost := uint64(rand.Int63n(testMaxCost)) + bv := n.node.RequestProcessed(0, n.index, testMaxCost, rcost) + if bv < testMaxCost { + n.waitUntil = now + mclock.AbsTime((testMaxCost-bv)*1001000/n.capacity) + } + //n.waitUntil = now + mclock.AbsTime(float64(testMaxCost)*1001000/float64(n.capacity)*(1-float64(bv)/float64(n.bufLimit))) + n.totalCost += rcost + return true +} diff --git a/les/freeclient.go b/les/freeclient.go index 5ee607be8f..d859337c26 100644 --- a/les/freeclient.go +++ b/les/freeclient.go @@ -14,12 +14,12 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( "io" "math" + "net" "sync" "time" @@ -44,12 +44,14 @@ import ( // value for the client. Currently the LES protocol manager uses IP addresses // (without port address) to identify clients. type freeClientPool struct { - db ethdb.Database - lock sync.Mutex - clock mclock.Clock - closed bool + db ethdb.Database + lock sync.Mutex + clock mclock.Clock + closed bool + removePeer func(string) connectedLimit, totalLimit int + freeClientCap uint64 addressMap map[string]*freeClientPoolEntry connPool, disconnPool *prque.Prque @@ -64,15 +66,16 @@ const ( ) // newFreeClientPool creates a new free client pool -func newFreeClientPool(db ethdb.Database, connectedLimit, totalLimit int, clock mclock.Clock) *freeClientPool { +func newFreeClientPool(db ethdb.Database, freeClientCap uint64, totalLimit int, clock mclock.Clock, removePeer func(string)) *freeClientPool { pool := &freeClientPool{ - db: db, - clock: clock, - addressMap: make(map[string]*freeClientPoolEntry), - connPool: prque.New(poolSetIndex), - disconnPool: prque.New(poolSetIndex), - connectedLimit: connectedLimit, - totalLimit: totalLimit, + db: db, + clock: clock, + addressMap: make(map[string]*freeClientPoolEntry), + connPool: prque.New(poolSetIndex), + disconnPool: prque.New(poolSetIndex), + freeClientCap: freeClientCap, + totalLimit: totalLimit, + removePeer: removePeer, } pool.loadFromDb() return pool @@ -85,22 +88,34 @@ func (f *freeClientPool) stop() { f.lock.Unlock() } +// registerPeer implements clientPool +func (f *freeClientPool) registerPeer(p *peer) { + if addr, ok := p.RemoteAddr().(*net.TCPAddr); ok { + if !f.connect(addr.IP.String(), p.id) { + f.removePeer(p.id) + } + } +} + // connect should be called after a successful handshake. If the connection was // rejected, there is no need to call disconnect. -// -// Note: the disconnectFn callback should not block. -func (f *freeClientPool) connect(address string, disconnectFn func()) bool { +func (f *freeClientPool) connect(address, id string) bool { f.lock.Lock() defer f.lock.Unlock() if f.closed { return false } + + if f.connectedLimit == 0 { + log.Debug("Client rejected", "address", address) + return false + } e := f.addressMap[address] now := f.clock.Now() var recentUsage int64 if e == nil { - e = &freeClientPoolEntry{address: address, index: -1} + e = &freeClientPoolEntry{address: address, index: -1, id: id} f.addressMap[address] = e } else { if e.connected { @@ -115,12 +130,7 @@ func (f *freeClientPool) connect(address string, disconnectFn func()) bool { i := f.connPool.PopItem().(*freeClientPoolEntry) if e.linUsage+int64(connectedBias)-i.linUsage < 0 { // kick it out and accept the new client - f.connPool.Remove(i.index) - f.calcLogUsage(i, now) - i.connected = false - f.disconnPool.Push(i, -i.logUsage) - log.Debug("Client kicked out", "address", i.address) - i.disconnectFn() + f.dropClient(i, now) } else { // keep the old client and reject the new one f.connPool.Push(i, i.linUsage) @@ -130,7 +140,7 @@ func (f *freeClientPool) connect(address string, disconnectFn func()) bool { } f.disconnPool.Remove(e.index) e.connected = true - e.disconnectFn = disconnectFn + e.id = id f.connPool.Push(e, e.linUsage) if f.connPool.Size()+f.disconnPool.Size() > f.totalLimit { f.disconnPool.Pop() @@ -139,6 +149,13 @@ func (f *freeClientPool) connect(address string, disconnectFn func()) bool { return true } +// unregisterPeer implements clientPool +func (f *freeClientPool) unregisterPeer(p *peer) { + if addr, ok := p.RemoteAddr().(*net.TCPAddr); ok { + f.disconnect(addr.IP.String()) + } +} + // disconnect should be called when a connection is terminated. If the disconnection // was initiated by the pool itself using disconnectFn then calling disconnect is // not necessary but permitted. @@ -163,6 +180,34 @@ func (f *freeClientPool) disconnect(address string) { log.Debug("Client disconnected", "address", address) } +// setConnLimit sets the maximum number of free client slots and also drops +// some peers if necessary +func (f *freeClientPool) setLimits(count int, totalCap uint64) { + f.lock.Lock() + defer f.lock.Unlock() + + f.connectedLimit = int(totalCap / f.freeClientCap) + if count < f.connectedLimit { + f.connectedLimit = count + } + now := mclock.Now() + for f.connPool.Size() > f.connectedLimit { + i := f.connPool.PopItem().(*freeClientPoolEntry) + f.dropClient(i, now) + } +} + +// dropClient disconnects a client and also moves it from the connected to the +// disconnected pool +func (f *freeClientPool) dropClient(i *freeClientPoolEntry, now mclock.AbsTime) { + f.connPool.Remove(i.index) + f.calcLogUsage(i, now) + i.connected = false + f.disconnPool.Push(i, -i.logUsage) + log.Debug("Client kicked out", "address", i.address) + f.removePeer(i.id) +} + // logOffset calculates the time-dependent offset for the logarithmic // representation of recent usage func (f *freeClientPool) logOffset(now mclock.AbsTime) int64 { @@ -245,7 +290,7 @@ func (f *freeClientPool) saveToDb() { // even though they are close to each other at any time they may wrap around int64 // limits over time. Comparison should be performed accordingly. type freeClientPoolEntry struct { - address string + address, id string connected bool disconnectFn func() linUsage, logUsage int64 diff --git a/les/freeclient_test.go b/les/freeclient_test.go index e95abc7aad..dc8b51a8d2 100644 --- a/les/freeclient_test.go +++ b/les/freeclient_test.go @@ -14,13 +14,12 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package les import ( "fmt" "math/rand" + "strconv" "testing" "time" @@ -44,32 +43,38 @@ const testFreeClientPoolTicks = 500000 func testFreeClientPool(t *testing.T, connLimit, clientCount int) { var ( - clock mclock.Simulated - db = ethdb.NewMemDatabase() - pool = newFreeClientPool(db, connLimit, 10000, &clock) - connected = make([]bool, clientCount) - connTicks = make([]int, clientCount) - disconnCh = make(chan int, clientCount) - ) - peerId := func(i int) string { - return fmt.Sprintf("test peer #%d", i) - } - disconnFn := func(i int) func() { - return func() { + clock mclock.Simulated + db = ethdb.NewMemDatabase() + connected = make([]bool, clientCount) + connTicks = make([]int, clientCount) + disconnCh = make(chan int, clientCount) + peerAddress = func(i int) string { + return fmt.Sprintf("addr #%d", i) + } + peerId = func(i int) string { + return fmt.Sprintf("id #%d", i) + } + disconnFn = func(id string) { + i, err := strconv.Atoi(id[4:]) + if err != nil { + panic(err) + } disconnCh <- i } - } + pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn) + ) + pool.setLimits(connLimit, uint64(connLimit)) // pool should accept new peers up to its connected limit for i := 0; i < connLimit; i++ { - if pool.connect(peerId(i), disconnFn(i)) { + if pool.connect(peerAddress(i), peerId(i)) { connected[i] = true } else { t.Fatalf("Test peer #%d rejected", i) } } // since all accepted peers are new and should not be kicked out, the next one should be rejected - if pool.connect(peerId(connLimit), disconnFn(connLimit)) { + if pool.connect(peerAddress(connLimit), peerId(connLimit)) { connected[connLimit] = true t.Fatalf("Peer accepted over connected limit") } @@ -80,11 +85,11 @@ func testFreeClientPool(t *testing.T, connLimit, clientCount int) { i := rand.Intn(clientCount) if connected[i] { - pool.disconnect(peerId(i)) + pool.disconnect(peerAddress(i)) connected[i] = false connTicks[i] += tickCounter } else { - if pool.connect(peerId(i), disconnFn(i)) { + if pool.connect(peerAddress(i), peerId(i)) { connected[i] = true connTicks[i] -= tickCounter } @@ -93,7 +98,7 @@ func testFreeClientPool(t *testing.T, connLimit, clientCount int) { for { select { case i := <-disconnCh: - pool.disconnect(peerId(i)) + pool.disconnect(peerAddress(i)) if connected[i] { connTicks[i] += tickCounter connected[i] = false @@ -119,20 +124,21 @@ func testFreeClientPool(t *testing.T, connLimit, clientCount int) { } // a previously unknown peer should be accepted now - if !pool.connect("newPeer", func() {}) { + if !pool.connect("newAddr", "newId") { t.Fatalf("Previously unknown peer rejected") } // close and restart pool pool.stop() - pool = newFreeClientPool(db, connLimit, 10000, &clock) + pool = newFreeClientPool(db, 1, 10000, &clock, disconnFn) + pool.setLimits(connLimit, uint64(connLimit)) // try connecting all known peers (connLimit should be filled up) for i := 0; i < clientCount; i++ { - pool.connect(peerId(i), func() {}) + pool.connect(peerAddress(i), peerId(i)) } // expect pool to remember known nodes and kick out one of them to accept a new one - if !pool.connect("newPeer2", func() {}) { + if !pool.connect("newAddr2", "newId2") { t.Errorf("Previously unknown peer rejected after restarting pool") } pool.stop() diff --git a/les/handler.go b/les/handler.go index 680e115b04..0352f5b039 100644 --- a/les/handler.go +++ b/les/handler.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( @@ -22,12 +21,10 @@ import ( "encoding/json" "fmt" "math/big" - "net" "sync" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -90,21 +87,21 @@ type txPool interface { } type ProtocolManager struct { - lightSync bool - txpool txPool - txrelay *LesTxRelay - networkId uint64 - chainConfig *params.ChainConfig - iConfig *light.IndexerConfig - blockchain BlockChain - chainDb ethdb.Database - odr *LesOdr - server *LesServer - serverPool *serverPool - clientPool *freeClientPool - lesTopic discv5.Topic - reqDist *requestDistributor - retriever *retrieveManager + lightSync bool + txpool txPool + txrelay *LesTxRelay + networkId uint64 + chainConfig *params.ChainConfig + iConfig *light.IndexerConfig + blockchain BlockChain + chainDb ethdb.Database + odr *LesOdr + server *LesServer + serverPool *serverPool + lesTopic discv5.Topic + reqDist *requestDistributor + retriever *retrieveManager + servingQueue *servingQueue downloader *downloader.Downloader fetcher *lightFetcher @@ -165,6 +162,8 @@ func NewProtocolManager( if odr != nil { manager.retriever = odr.retriever manager.reqDist = odr.retriever.dist + } else { + manager.servingQueue = newServingQueue(int64(time.Millisecond * 10)) } if ulcConfig != nil { @@ -181,7 +180,6 @@ func NewProtocolManager( manager.peers.notify((*downloaderPeerNotify)(manager)) manager.fetcher = newLightFetcher(manager) } - return manager, nil } @@ -192,11 +190,9 @@ func (pm *ProtocolManager) removePeer(id string) { func (pm *ProtocolManager) Start(maxPeers int) { pm.maxPeers = maxPeers - if pm.lightSync { go pm.syncer() } else { - pm.clientPool = newFreeClientPool(pm.chainDb, maxPeers, 10000, mclock.System{}) go func() { for range pm.newPeerCh { } @@ -214,8 +210,9 @@ func (pm *ProtocolManager) Stop() { pm.noMorePeers <- struct{}{} close(pm.quitSync) // quits syncer, fetcher - if pm.clientPool != nil { - pm.clientPool.stop() + + if pm.servingQueue != nil { + pm.servingQueue.stop() } // Disconnect existing sessions. @@ -286,17 +283,8 @@ func (pm *ProtocolManager) handle(p *peer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } - - if !pm.lightSync && !p.Peer.Info().Network.Trusted { - addr, ok := p.RemoteAddr().(*net.TCPAddr) - // test peer address is not a tcp address, don't use client pool if can not typecast - if ok { - id := addr.IP.String() - if !pm.clientPool.connect(id, func() { go pm.removePeer(p.id) }) { - return p2p.DiscTooManyPeers - } - defer pm.clientPool.disconnect(id) - } + if p.fcClient != nil { + defer p.fcClient.Disconnect() } if rw, ok := p.rw.(*meteredMsgReadWriter); ok { @@ -309,9 +297,6 @@ func (pm *ProtocolManager) handle(p *peer) error { return err } defer func() { - if pm.server != nil && pm.server.fcManager != nil && p.fcClient != nil { - p.fcClient.Remove(pm.server.fcManager) - } pm.removePeer(p.id) }() @@ -329,31 +314,18 @@ func (pm *ProtocolManager) handle(p *peer) error { } } - stop := make(chan struct{}) - defer close(stop) - go func() { - // new block announce loop - for { - select { - case announce := <-p.announceChn: - p.SendAnnounce(announce) - case <-stop: - return - } - } - }() - // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { p.Log().Debug("Light Ethereum message handling failed", "err", err) + if p.fcServer != nil { + p.fcServer.DumpLogs() + } return err } } } -var reqList = []uint64{GetBlockHeadersMsg, GetBlockBodiesMsg, GetCodeMsg, GetReceiptsMsg, GetProofsV1Msg, SendTxMsg, SendTxV2Msg, GetTxStatusMsg, GetHeaderProofsMsg, GetProofsV2Msg, GetHelperTrieProofsMsg} - // handleMsg is invoked whenever an inbound message is received from a remote // peer. The remote connection is torn down upon returning any error. func (pm *ProtocolManager) handleMsg(p *peer) error { @@ -364,22 +336,31 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.Log().Trace("Light Ethereum message arrived", "code", msg.Code, "bytes", msg.Size) - costs := p.fcCosts[msg.Code] - reject := func(reqCnt, maxCnt uint64) bool { + p.responseCount++ + responseCount := p.responseCount + var ( + maxCost uint64 + task *servingTask + ) + + accept := func(reqID, reqCnt, maxCnt uint64) bool { + if reqCnt == 0 { + return false + } if p.fcClient == nil || reqCnt > maxCnt { - return true + return false } - bufValue, _ := p.fcClient.AcceptRequest() - cost := costs.baseCost + reqCnt*costs.reqCost - if cost > pm.server.defParams.BufLimit { - cost = pm.server.defParams.BufLimit + maxCost = p.fcCosts.getCost(msg.Code, reqCnt) + + if accepted, bufShort, servingPriority := p.fcClient.AcceptRequest(reqID, responseCount, maxCost); !accepted { + if bufShort > 0 { + p.Log().Error("Request came too early", "remaining", common.PrettyDuration(time.Duration(bufShort*1000000/p.fcParams.MinRecharge))) + } + return false + } else { + task = pm.servingQueue.newTask(servingPriority) } - if cost > bufValue { - recharge := time.Duration((cost - bufValue) * 1000000 / pm.server.defParams.MinRecharge) - p.Log().Error("Request came too early", "recharge", common.PrettyDuration(recharge)) - return true - } - return false + return task.start() } if msg.Size > ProtocolMaxMsgSize { @@ -389,6 +370,31 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { var deliverMsg *Msg + sendResponse := func(reqID, amount uint64, reply *reply, servingTime uint64) { + p.responseLock.Lock() + defer p.responseLock.Unlock() + + var replySize uint32 + if reply != nil { + replySize = reply.size() + } + var realCost uint64 + if pm.server.costTracker != nil { + realCost = pm.server.costTracker.realCost(servingTime, msg.Size, replySize) + pm.server.costTracker.updateStats(msg.Code, amount, servingTime, realCost) + } else { + realCost = maxCost + } + bv := p.fcClient.RequestProcessed(reqID, responseCount, maxCost, realCost) + if reply != nil { + p.queueSend(func() { + if err := reply.send(bv); err != nil { + p.errCh <- err + } + }) + } + } + // Handle the message depending on its contents switch msg.Code { case StatusMsg: @@ -399,25 +405,33 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Block header query, collect the requested headers and reply case AnnounceMsg: p.Log().Trace("Received announce message") - if p.announceType == announceTypeNone { - return errResp(ErrUnexpectedResponse, "") - } var req announceData if err := msg.Decode(&req); err != nil { return errResp(ErrDecode, "%v: %v", msg, err) } - if p.announceType == announceTypeSigned { - if err := req.checkSignature(p.ID()); err != nil { - p.Log().Trace("Invalid announcement signature", "err", err) - return err - } - p.Log().Trace("Valid announcement signature") + update, size := req.Update.decode() + if p.rejectUpdate(size) { + return errResp(ErrRequestRejected, "") } + p.updateFlowControl(update) - p.Log().Trace("Announce message content", "number", req.Number, "hash", req.Hash, "td", req.Td, "reorg", req.ReorgDepth) - if pm.fetcher != nil { - pm.fetcher.announce(p, &req) + if req.Hash != (common.Hash{}) { + if p.announceType == announceTypeNone { + return errResp(ErrUnexpectedResponse, "") + } + if p.announceType == announceTypeSigned { + if err := req.checkSignature(p.ID(), update); err != nil { + p.Log().Trace("Invalid announcement signature", "err", err) + return err + } + p.Log().Trace("Valid announcement signature") + } + + p.Log().Trace("Announce message content", "number", req.Number, "hash", req.Hash, "td", req.Td, "reorg", req.ReorgDepth) + if pm.fetcher != nil { + pm.fetcher.announce(p, &req) + } } case GetBlockHeadersMsg: @@ -432,93 +446,94 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } query := req.Query - if reject(query.Amount, MaxHeaderFetch) { + if !accept(req.ReqID, query.Amount, MaxHeaderFetch) { return errResp(ErrRequestRejected, "") } + go func() { + hashMode := query.Origin.Hash != (common.Hash{}) + first := true + maxNonCanonical := uint64(100) - hashMode := query.Origin.Hash != (common.Hash{}) - first := true - maxNonCanonical := uint64(100) - - // Gather headers until the fetch or network limits is reached - var ( - bytes common.StorageSize - headers []*types.Header - unknown bool - ) - for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit { - // Retrieve the next header satisfying the query - var origin *types.Header - if hashMode { - if first { - first = false - origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) - if origin != nil { - query.Origin.Number = origin.Number.Uint64() + // Gather headers until the fetch or network limits is reached + var ( + bytes common.StorageSize + headers []*types.Header + unknown bool + ) + for !unknown && len(headers) < int(query.Amount) && bytes < softResponseLimit { + if !first && !task.waitOrStop() { + return + } + // Retrieve the next header satisfying the query + var origin *types.Header + if hashMode { + if first { + origin = pm.blockchain.GetHeaderByHash(query.Origin.Hash) + if origin != nil { + query.Origin.Number = origin.Number.Uint64() + } + } else { + origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) } } else { - origin = pm.blockchain.GetHeader(query.Origin.Hash, query.Origin.Number) + origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) } - } else { - origin = pm.blockchain.GetHeaderByNumber(query.Origin.Number) - } - if origin == nil { - break - } - headers = append(headers, origin) - bytes += estHeaderRlpSize + if origin == nil { + break + } + headers = append(headers, origin) + bytes += estHeaderRlpSize - // Advance to the next header of the query - switch { - case hashMode && query.Reverse: - // Hash based traversal towards the genesis block - ancestor := query.Skip + 1 - if ancestor == 0 { - unknown = true - } else { - query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) - unknown = (query.Origin.Hash == common.Hash{}) - } - case hashMode && !query.Reverse: - // Hash based traversal towards the leaf block - var ( - current = origin.Number.Uint64() - next = current + query.Skip + 1 - ) - if next <= current { - infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") - p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) - unknown = true - } else { - if header := pm.blockchain.GetHeaderByNumber(next); header != nil { - nextHash := header.Hash() - expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) - if expOldHash == query.Origin.Hash { - query.Origin.Hash, query.Origin.Number = nextHash, next + // Advance to the next header of the query + switch { + case hashMode && query.Reverse: + // Hash based traversal towards the genesis block + ancestor := query.Skip + 1 + if ancestor == 0 { + unknown = true + } else { + query.Origin.Hash, query.Origin.Number = pm.blockchain.GetAncestor(query.Origin.Hash, query.Origin.Number, ancestor, &maxNonCanonical) + unknown = (query.Origin.Hash == common.Hash{}) + } + case hashMode && !query.Reverse: + // Hash based traversal towards the leaf block + var ( + current = origin.Number.Uint64() + next = current + query.Skip + 1 + ) + if next <= current { + infos, _ := json.MarshalIndent(p.Peer.Info(), "", " ") + p.Log().Warn("GetBlockHeaders skip overflow attack", "current", current, "skip", query.Skip, "next", next, "attacker", infos) + unknown = true + } else { + if header := pm.blockchain.GetHeaderByNumber(next); header != nil { + nextHash := header.Hash() + expOldHash, _ := pm.blockchain.GetAncestor(nextHash, next, query.Skip+1, &maxNonCanonical) + if expOldHash == query.Origin.Hash { + query.Origin.Hash, query.Origin.Number = nextHash, next + } else { + unknown = true + } } else { unknown = true } + } + case query.Reverse: + // Number based traversal towards the genesis block + if query.Origin.Number >= query.Skip+1 { + query.Origin.Number -= query.Skip + 1 } else { unknown = true } - } - case query.Reverse: - // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { - unknown = true - } - case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 + case !query.Reverse: + // Number based traversal towards the leaf block + query.Origin.Number += query.Skip + 1 + } + first = false } - } - - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + query.Amount*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, query.Amount, rcost) - return p.SendBlockHeaders(req.ReqID, bv, headers) + sendResponse(req.ReqID, query.Amount, p.ReplyBlockHeaders(req.ReqID, headers), task.done()) + }() case BlockHeadersMsg: if pm.downloader == nil { @@ -534,7 +549,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) if pm.fetcher != nil && pm.fetcher.requestedID(resp.ReqID) { pm.fetcher.deliverHeaders(p, resp.ReqID, resp.Headers) } else { @@ -560,24 +575,27 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { bodies []rlp.RawValue ) reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxBodyFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxBodyFetch) { return errResp(ErrRequestRejected, "") } - for _, hash := range req.Hashes { - if bytes >= softResponseLimit { - break - } - // Retrieve the requested block body, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { - if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { - bodies = append(bodies, data) - bytes += len(data) + go func() { + for i, hash := range req.Hashes { + if i != 0 && !task.waitOrStop() { + return + } + if bytes >= softResponseLimit { + break + } + // Retrieve the requested block body, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + if data := rawdb.ReadBodyRLP(pm.chainDb, hash, *number); len(data) != 0 { + bodies = append(bodies, data) + bytes += len(data) + } } } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendBlockBodiesRLP(req.ReqID, bv, bodies) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyBlockBodiesRLP(req.ReqID, bodies), task.done()) + }() case BlockBodiesMsg: if pm.odr == nil { @@ -593,7 +611,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgBlockBodies, ReqID: resp.ReqID, @@ -616,33 +634,36 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { data [][]byte ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxCodeFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxCodeFetch) { return errResp(ErrRequestRejected, "") } - for _, req := range req.Reqs { - // Retrieve the requested state entry, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) - if err != nil { - continue - } - code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) + go func() { + for i, req := range req.Reqs { + if i != 0 && !task.waitOrStop() { + return + } + // Retrieve the requested state entry, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() + if err != nil { + continue + } + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + code, _ := statedb.Database().TrieDB().Node(common.BytesToHash(account.CodeHash)) - data = append(data, code) - if bytes += len(code); bytes >= softResponseLimit { - break + data = append(data, code) + if bytes += len(code); bytes >= softResponseLimit { + break + } } } } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendCode(req.ReqID, bv, data) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyCode(req.ReqID, data), task.done()) + }() case CodeMsg: if pm.odr == nil { @@ -658,7 +679,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgCode, ReqID: resp.ReqID, @@ -681,34 +702,37 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { receipts []rlp.RawValue ) reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxReceiptFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxReceiptFetch) { return errResp(ErrRequestRejected, "") } - for _, hash := range req.Hashes { - if bytes >= softResponseLimit { - break - } - // Retrieve the requested block's receipts, skipping if unknown to us - var results types.Receipts - if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { - results = rawdb.ReadReceipts(pm.chainDb, hash, *number) - } - if results == nil { - if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue + go func() { + for i, hash := range req.Hashes { + if i != 0 && !task.waitOrStop() { + return + } + if bytes >= softResponseLimit { + break + } + // Retrieve the requested block's receipts, skipping if unknown to us + var results types.Receipts + if number := rawdb.ReadHeaderNumber(pm.chainDb, hash); number != nil { + results = rawdb.ReadReceipts(pm.chainDb, hash, *number) + } + if results == nil { + if header := pm.blockchain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { + continue + } + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(results); err != nil { + log.Error("Failed to encode receipt", "err", err) + } else { + receipts = append(receipts, encoded) + bytes += len(encoded) } } - // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(results); err != nil { - log.Error("Failed to encode receipt", "err", err) - } else { - receipts = append(receipts, encoded) - bytes += len(encoded) - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendReceiptsRLP(req.ReqID, bv, receipts) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyReceiptsRLP(req.ReqID, receipts), task.done()) + }() case ReceiptsMsg: if pm.odr == nil { @@ -724,7 +748,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgReceipts, ReqID: resp.ReqID, @@ -747,42 +771,45 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { proofs proofsData ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxProofsFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxProofsFetch) { return errResp(ErrRequestRejected, "") } - for _, req := range req.Reqs { - // Retrieve the requested state entry, stopping if enough was found - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, err := pm.blockchain.State() - if err != nil { - continue - } - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + go func() { + for i, req := range req.Reqs { + if i != 0 && !task.waitOrStop() { + return + } + // Retrieve the requested state entry, stopping if enough was found + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, err := pm.blockchain.State() if err != nil { continue } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(header.Root) - } - if trie != nil { - var proof light.NodeList - trie.Prove(req.Key, 0, &proof) + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, header.Root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(header.Root) + } + if trie != nil { + var proof light.NodeList + trie.Prove(req.Key, 0, &proof) - proofs = append(proofs, proof) - if bytes += proof.DataSize(); bytes >= softResponseLimit { - break + proofs = append(proofs, proof) + if bytes += proof.DataSize(); bytes >= softResponseLimit { + break + } } } } } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofs(req.ReqID, bv, proofs) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyProofs(req.ReqID, proofs), task.done()) + }() case GetProofsV2Msg: p.Log().Trace("Received les/2 proofs request") @@ -801,50 +828,53 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { root common.Hash ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxProofsFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxProofsFetch) { return errResp(ErrRequestRejected, "") } + go func() { - nodes := light.NewNodeSet() + nodes := light.NewNodeSet() - for _, req := range req.Reqs { - // Look up the state belonging to the request - if statedb == nil || req.BHash != lastBHash { - statedb, root, lastBHash = nil, common.Hash{}, req.BHash + for i, req := range req.Reqs { + if i != 0 && !task.waitOrStop() { + return + } + // Look up the state belonging to the request + if statedb == nil || req.BHash != lastBHash { + statedb, root, lastBHash = nil, common.Hash{}, req.BHash - if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { - if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { - statedb, _ = pm.blockchain.State() - root = header.Root + if number := rawdb.ReadHeaderNumber(pm.chainDb, req.BHash); number != nil { + if header := rawdb.ReadHeader(pm.chainDb, req.BHash, *number); header != nil { + statedb, _ = pm.blockchain.State() + root = header.Root + } } } - } - if statedb == nil { - continue - } - // Pull the account or storage trie of the request - var trie state.Trie - if len(req.AccKey) > 0 { - account, err := pm.getAccount(statedb, root, common.BytesToHash(req.AccKey)) - if err != nil { + if statedb == nil { continue } - trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) - } else { - trie, _ = statedb.Database().OpenTrie(root) + // Pull the account or storage trie of the request + var trie state.Trie + if len(req.AccKey) > 0 { + account, err := pm.getAccount(statedb, root, common.BytesToHash(req.AccKey)) + if err != nil { + continue + } + trie, _ = statedb.Database().OpenStorageTrie(common.BytesToHash(req.AccKey), account.Root) + } else { + trie, _ = statedb.Database().OpenTrie(root) + } + if trie == nil { + continue + } + // Prove the user's request from the account or stroage trie + trie.Prove(req.Key, req.FromLevel, nodes) + if nodes.DataSize() >= softResponseLimit { + break + } } - if trie == nil { - continue - } - // Prove the user's request from the account or stroage trie - trie.Prove(req.Key, req.FromLevel, nodes) - if nodes.DataSize() >= softResponseLimit { - break - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendProofsV2(req.ReqID, bv, nodes.NodeList()) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyProofsV2(req.ReqID, nodes.NodeList()), task.done()) + }() case ProofsV1Msg: if pm.odr == nil { @@ -860,7 +890,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgProofsV1, ReqID: resp.ReqID, @@ -881,7 +911,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgProofsV2, ReqID: resp.ReqID, @@ -904,34 +934,37 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { proofs []ChtResp ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } - trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) - for _, req := range req.Reqs { - if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { - sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) - if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { - trie, err := trie.New(root, trieDb) - if err != nil { - continue - } - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], req.BlockNum) + go func() { + trieDb := trie.NewDatabase(ethdb.NewTable(pm.chainDb, light.ChtTablePrefix)) + for i, req := range req.Reqs { + if i != 0 && !task.waitOrStop() { + return + } + if header := pm.blockchain.GetHeaderByNumber(req.BlockNum); header != nil { + sectionHead := rawdb.ReadCanonicalHash(pm.chainDb, req.ChtNum*pm.iConfig.ChtSize-1) + if root := light.GetChtRoot(pm.chainDb, req.ChtNum-1, sectionHead); root != (common.Hash{}) { + trie, err := trie.New(root, trieDb) + if err != nil { + continue + } + var encNumber [8]byte + binary.BigEndian.PutUint64(encNumber[:], req.BlockNum) - var proof light.NodeList - trie.Prove(encNumber[:], 0, &proof) + var proof light.NodeList + trie.Prove(encNumber[:], 0, &proof) - proofs = append(proofs, ChtResp{Header: header, Proof: proof}) - if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { - break + proofs = append(proofs, ChtResp{Header: header, Proof: proof}) + if bytes += proof.DataSize() + estHeaderRlpSize; bytes >= softResponseLimit { + break + } } } } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHeaderProofs(req.ReqID, bv, proofs) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyHeaderProofs(req.ReqID, proofs), task.done()) + }() case GetHelperTrieProofsMsg: p.Log().Trace("Received helper trie proof request") @@ -949,50 +982,53 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { auxData [][]byte ) reqCnt := len(req.Reqs) - if reject(uint64(reqCnt), MaxHelperTrieProofsFetch) { + if !accept(req.ReqID, uint64(reqCnt), MaxHelperTrieProofsFetch) { return errResp(ErrRequestRejected, "") } + go func() { - var ( - lastIdx uint64 - lastType uint - root common.Hash - auxTrie *trie.Trie - ) - nodes := light.NewNodeSet() - for _, req := range req.Reqs { - if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { - auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx + var ( + lastIdx uint64 + lastType uint + root common.Hash + auxTrie *trie.Trie + ) + nodes := light.NewNodeSet() + for i, req := range req.Reqs { + if i != 0 && !task.waitOrStop() { + return + } + if auxTrie == nil || req.Type != lastType || req.TrieIdx != lastIdx { + auxTrie, lastType, lastIdx = nil, req.Type, req.TrieIdx - var prefix string - if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { - auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) + var prefix string + if root, prefix = pm.getHelperTrie(req.Type, req.TrieIdx); root != (common.Hash{}) { + auxTrie, _ = trie.New(root, trie.NewDatabase(ethdb.NewTable(pm.chainDb, prefix))) + } } - } - if req.AuxReq == auxRoot { - var data []byte - if root != (common.Hash{}) { - data = root[:] - } - auxData = append(auxData, data) - auxBytes += len(data) - } else { - if auxTrie != nil { - auxTrie.Prove(req.Key, req.FromLevel, nodes) - } - if req.AuxReq != 0 { - data := pm.getHelperTrieAuxData(req) + if req.AuxReq == auxRoot { + var data []byte + if root != (common.Hash{}) { + data = root[:] + } auxData = append(auxData, data) auxBytes += len(data) + } else { + if auxTrie != nil { + auxTrie.Prove(req.Key, req.FromLevel, nodes) + } + if req.AuxReq != 0 { + data := pm.getHelperTrieAuxData(req) + auxData = append(auxData, data) + auxBytes += len(data) + } + } + if nodes.DataSize()+auxBytes >= softResponseLimit { + break } } - if nodes.DataSize()+auxBytes >= softResponseLimit { - break - } - } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - return p.SendHelperTrieProofs(req.ReqID, bv, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyHelperTrieProofs(req.ReqID, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}), task.done()) + }() case HeaderProofsMsg: if pm.odr == nil { @@ -1007,7 +1043,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgHeaderProofs, ReqID: resp.ReqID, @@ -1028,7 +1064,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) deliverMsg = &Msg{ MsgType: MsgHelperTrieProofs, ReqID: resp.ReqID, @@ -1045,13 +1081,18 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(txs) - if reject(uint64(reqCnt), MaxTxSend) { + if !accept(0, uint64(reqCnt), MaxTxSend) { return errResp(ErrRequestRejected, "") } - pm.txpool.AddRemotes(txs) - - _, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) + go func() { + for i, tx := range txs { + if i != 0 && !task.waitOrStop() { + return + } + pm.txpool.AddRemotes([]*types.Transaction{tx}) + } + sendResponse(0, uint64(reqCnt), nil, task.done()) + }() case SendTxV2Msg: if pm.txpool == nil { @@ -1066,29 +1107,27 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(req.Txs) - if reject(uint64(reqCnt), MaxTxSend) { + if !accept(req.ReqID, uint64(reqCnt), MaxTxSend) { return errResp(ErrRequestRejected, "") } - - hashes := make([]common.Hash, len(req.Txs)) - for i, tx := range req.Txs { - hashes[i] = tx.Hash() - } - stats := pm.txStatus(hashes) - for i, stat := range stats { - if stat.Status == core.TxStatusUnknown { - if errs := pm.txpool.AddRemotes([]*types.Transaction{req.Txs[i]}); errs[0] != nil { - stats[i].Error = errs[0].Error() - continue + go func() { + stats := make([]txStatus, len(req.Txs)) + for i, tx := range req.Txs { + if i != 0 && !task.waitOrStop() { + return + } + hash := tx.Hash() + stats[i] = pm.txStatus(hash) + if stats[i].Status == core.TxStatusUnknown { + if errs := pm.txpool.AddRemotes([]*types.Transaction{tx}); errs[0] != nil { + stats[i].Error = errs[0].Error() + continue + } + stats[i] = pm.txStatus(hash) } - stats[i] = pm.txStatus([]common.Hash{hashes[i]})[0] } - } - - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - - return p.SendTxStatus(req.ReqID, bv, stats) + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyTxStatus(req.ReqID, stats), task.done()) + }() case GetTxStatusMsg: if pm.txpool == nil { @@ -1103,13 +1142,19 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } reqCnt := len(req.Hashes) - if reject(uint64(reqCnt), MaxTxStatus) { + if !accept(req.ReqID, uint64(reqCnt), MaxTxStatus) { return errResp(ErrRequestRejected, "") } - bv, rcost := p.fcClient.RequestProcessed(costs.baseCost + uint64(reqCnt)*costs.reqCost) - pm.server.fcCostStats.update(msg.Code, uint64(reqCnt), rcost) - - return p.SendTxStatus(req.ReqID, bv, pm.txStatus(req.Hashes)) + go func() { + stats := make([]txStatus, len(req.Hashes)) + for i, hash := range req.Hashes { + if i != 0 && !task.waitOrStop() { + return + } + stats[i] = pm.txStatus(hash) + } + sendResponse(req.ReqID, uint64(reqCnt), p.ReplyTxStatus(req.ReqID, stats), task.done()) + }() case TxStatusMsg: if pm.odr == nil { @@ -1125,7 +1170,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } - p.fcServer.GotReply(resp.ReqID, resp.BV) + p.fcServer.ReceivedReply(resp.ReqID, resp.BV) default: p.Log().Trace("Received unknown message", "code", msg.Code) @@ -1185,21 +1230,17 @@ func (pm *ProtocolManager) getHelperTrieAuxData(req HelperTrieReq) []byte { return nil } -func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus { - stats := make([]txStatus, len(hashes)) - for i, stat := range pm.txpool.Status(hashes) { - // Save the status we've got from the transaction pool - stats[i].Status = stat - - // If the transaction is unknown to the pool, try looking it up locally - if stat == core.TxStatusUnknown { - if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hashes[i]); tx != nil { - stats[i].Status = core.TxStatusIncluded - stats[i].Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} - } +func (pm *ProtocolManager) txStatus(hash common.Hash) txStatus { + var stat txStatus + stat.Status = pm.txpool.Status([]common.Hash{hash})[0] + // If the transaction is unknown to the pool, try looking it up locally + if stat.Status == core.TxStatusUnknown { + if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hash); tx != nil { + stat.Status = core.TxStatusIncluded + stat.Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex} } } - return stats + return stat } // isULCEnabled returns true if we can use ULC @@ -1235,7 +1276,7 @@ func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, s request: func(dp distPeer) func() { peer := dp.(*peer) cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) + peer.fcServer.QueuedRequest(reqID, cost) return func() { peer.RequestHeadersByHash(reqID, cost, origin, amount, skip, reverse) } }, } @@ -1259,7 +1300,7 @@ func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip request: func(dp distPeer) func() { peer := dp.(*peer) cost := peer.GetRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueueRequest(reqID, cost) + peer.fcServer.QueuedRequest(reqID, cost) return func() { peer.RequestHeadersByNumber(reqID, cost, origin, amount, skip, reverse) } }, } diff --git a/les/helper_test.go b/les/helper_test.go index 02b1668c84..a8097708ee 100644 --- a/les/helper_test.go +++ b/les/helper_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -133,16 +134,6 @@ func testIndexers(db ethdb.Database, odr light.OdrBackend, iConfig *light.Indexe return chtIndexer, bloomIndexer, bloomTrieIndexer } -func testRCL() RequestCostList { - cl := make(RequestCostList, len(reqList)) - for i, code := range reqList { - cl[i].MsgCode = code - cl[i].BaseCost = 0 - cl[i].ReqCost = 0 - } - return cl -} - // newTestProtocolManager creates a new protocol manager for testing purposes, // with the given number of blocks already known, potential notification // channels for different events and relative chain indexers array. @@ -183,14 +174,14 @@ func newTestProtocolManager(lightSync bool, blocks int, generator func(int, *cor if !lightSync { srv := &LesServer{lesCommons: lesCommons{protocolManager: pm}} pm.server = srv + pm.servingQueue.setThreads(4) - srv.defParams = &flowcontrol.ServerParams{ + srv.defParams = flowcontrol.ServerParams{ BufLimit: testBufLimit, MinRecharge: 1, } - srv.fcManager = flowcontrol.NewClientManager(50, 10, 1000000000) - srv.fcCostStats = newCostStats(nil) + srv.fcManager = flowcontrol.NewClientManager(nil, &mclock.System{}) } pm.Start(1000) return pm, nil @@ -304,7 +295,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu expList = expList.add("txRelay", nil) expList = expList.add("flowControl/BL", testBufLimit) expList = expList.add("flowControl/MRR", uint64(1)) - expList = expList.add("flowControl/MRC", testRCL()) + expList = expList.add("flowControl/MRC", testCostList()) if err := p2p.ExpectMsg(p.app, StatusMsg, expList); err != nil { t.Fatalf("status recv: %v", err) @@ -313,7 +304,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, headNu t.Fatalf("status send: %v", err) } - p.fcServerParams = &flowcontrol.ServerParams{ + p.fcParams = flowcontrol.ServerParams{ BufLimit: testBufLimit, MinRecharge: 1, } @@ -375,7 +366,7 @@ func newClientServerEnv(t *testing.T, blocks int, protocol int, waitIndexers fun db, ldb := ethdb.NewMemDatabase(), ethdb.NewMemDatabase() peers, lPeers := newPeerSet(), newPeerSet() - dist := newRequestDistributor(lPeers, make(chan struct{})) + dist := newRequestDistributor(lPeers, make(chan struct{}), &mclock.System{}) rm := newRetrieveManager(lPeers, dist, nil) odr := NewLesOdr(ldb, light.TestClientIndexerConfig, rm) diff --git a/les/odr.go b/les/odr.go index f7592354de..5d98c66a90 100644 --- a/les/odr.go +++ b/les/odr.go @@ -117,7 +117,7 @@ func (odr *LesOdr) Retrieve(ctx context.Context, req light.OdrRequest) (err erro request: func(dp distPeer) func() { p := dp.(*peer) cost := lreq.GetCost(p) - p.fcServer.QueueRequest(reqID, cost) + p.fcServer.QueuedRequest(reqID, cost) return func() { lreq.Request(reqID, p) } }, } diff --git a/les/odr_requests.go b/les/odr_requests.go index 0f2e5dd9ec..7b7876762d 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package les import ( diff --git a/les/peer.go b/les/peer.go index 9ae94b20fe..8b506de629 100644 --- a/les/peer.go +++ b/les/peer.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( @@ -25,6 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/les/flowcontrol" @@ -42,6 +42,17 @@ var ( const maxResponseErrors = 50 // number of invalid responses tolerated (makes the protocol less brittle but still avoids spam) +// capacity limitation for parameter updates +const ( + allowedUpdateBytes = 100000 // initial/maximum allowed update size + allowedUpdateRate = time.Millisecond * 10 // time constant for recharging one byte of allowance +) + +// if the total encoded size of a sent transaction batch is over txSizeCostLimit +// per transaction then the request cost is calculated as proportional to the +// encoded size instead of the transaction count +const txSizeCostLimit = 0x10000 + const ( announceTypeNone = iota announceTypeSimple @@ -63,17 +74,24 @@ type peer struct { headInfo *announceData lock sync.RWMutex - announceChn chan announceData - sendQueue *execQueue + sendQueue *execQueue + + errCh chan error + // responseLock ensures that responses are queued in the same order as + // RequestProcessed is called + responseLock sync.Mutex + responseCount uint64 poolEntry *poolEntry hasBlock func(common.Hash, uint64, bool) bool responseErrors int + updateCounter uint64 + updateTime mclock.AbsTime - fcClient *flowcontrol.ClientNode // nil if the peer is server only - fcServer *flowcontrol.ServerNode // nil if the peer is client only - fcServerParams *flowcontrol.ServerParams - fcCosts requestCostTable + fcClient *flowcontrol.ClientNode // nil if the peer is server only + fcServer *flowcontrol.ServerNode // nil if the peer is client only + fcParams flowcontrol.ServerParams + fcCosts requestCostTable isTrusted bool isOnlyAnnounce bool @@ -83,16 +101,36 @@ func newPeer(version int, network uint64, isTrusted bool, p *p2p.Peer, rw p2p.Ms id := p.ID() return &peer{ - Peer: p, - rw: rw, - version: version, - network: network, - id: fmt.Sprintf("%x", id[:8]), - announceChn: make(chan announceData, 20), - isTrusted: isTrusted, + Peer: p, + rw: rw, + version: version, + network: network, + id: fmt.Sprintf("%x", id), + isTrusted: isTrusted, } } +// rejectUpdate returns true if a parameter update has to be rejected because +// the size and/or rate of updates exceed the capacity limitation +func (p *peer) rejectUpdate(size uint64) bool { + now := mclock.Now() + if p.updateCounter == 0 { + p.updateTime = now + } else { + dt := now - p.updateTime + r := uint64(dt / mclock.AbsTime(allowedUpdateRate)) + if p.updateCounter > r { + p.updateCounter -= r + p.updateTime += mclock.AbsTime(allowedUpdateRate * time.Duration(r)) + } else { + p.updateCounter = 0 + p.updateTime = now + } + } + p.updateCounter += size + return p.updateCounter > allowedUpdateBytes +} + func (p *peer) canQueue() bool { return p.sendQueue.canQueue() } @@ -147,6 +185,20 @@ func (p *peer) waitBefore(maxCost uint64) (time.Duration, float64) { return p.fcServer.CanSend(maxCost) } +// updateCapacity updates the request serving capacity assigned to a given client +// and also sends an announcement about the updated flow control parameters +func (p *peer) updateCapacity(cap uint64) { + p.responseLock.Lock() + defer p.responseLock.Unlock() + + p.fcParams = flowcontrol.ServerParams{MinRecharge: cap, BufLimit: cap * bufLimitRatio} + p.fcClient.UpdateParams(p.fcParams) + var kvList keyValueList + kvList = kvList.add("flowControl/MRR", cap) + kvList = kvList.add("flowControl/BL", cap*bufLimitRatio) + p.queueSend(func() { p.SendAnnounce(announceData{Update: kvList}) }) +} + func sendRequest(w p2p.MsgWriter, msgcode, reqID, cost uint64, data interface{}) error { type req struct { ReqID uint64 @@ -155,12 +207,27 @@ func sendRequest(w p2p.MsgWriter, msgcode, reqID, cost uint64, data interface{}) return p2p.Send(w, msgcode, req{reqID, data}) } -func sendResponse(w p2p.MsgWriter, msgcode, reqID, bv uint64, data interface{}) error { +// reply struct represents a reply with the actual data already RLP encoded and +// only the bv (buffer value) missing. This allows the serving mechanism to +// calculate the bv value which depends on the data size before sending the reply. +type reply struct { + w p2p.MsgWriter + msgcode, reqID uint64 + data rlp.RawValue +} + +// send sends the reply with the calculated buffer value +func (r *reply) send(bv uint64) error { type resp struct { ReqID, BV uint64 - Data interface{} + Data rlp.RawValue } - return p2p.Send(w, msgcode, resp{reqID, bv, data}) + return p2p.Send(r.w, r.msgcode, resp{r.reqID, bv, r.data}) +} + +// size returns the RLP encoded size of the message data +func (r *reply) size() uint32 { + return uint32(len(r.data)) } func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 { @@ -168,8 +235,34 @@ func (p *peer) GetRequestCost(msgcode uint64, amount int) uint64 { defer p.lock.RUnlock() cost := p.fcCosts[msgcode].baseCost + p.fcCosts[msgcode].reqCost*uint64(amount) - if cost > p.fcServerParams.BufLimit { - cost = p.fcServerParams.BufLimit + if cost > p.fcParams.BufLimit { + cost = p.fcParams.BufLimit + } + return cost +} + +func (p *peer) GetTxRelayCost(amount, size int) uint64 { + p.lock.RLock() + defer p.lock.RUnlock() + + var msgcode uint64 + switch p.version { + case lpv1: + msgcode = SendTxMsg + case lpv2: + msgcode = SendTxV2Msg + default: + panic(nil) + } + + cost := p.fcCosts[msgcode].baseCost + p.fcCosts[msgcode].reqCost*uint64(amount) + sizeCost := p.fcCosts[msgcode].baseCost + p.fcCosts[msgcode].reqCost*uint64(size)/txSizeCostLimit + if sizeCost > cost { + cost = sizeCost + } + + if cost > p.fcParams.BufLimit { + cost = p.fcParams.BufLimit } return cost } @@ -188,52 +281,61 @@ func (p *peer) SendAnnounce(request announceData) error { return p2p.Send(p.rw, AnnounceMsg, request) } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *peer) SendBlockHeaders(reqID, bv uint64, headers []*types.Header) error { - return sendResponse(p.rw, BlockHeadersMsg, reqID, bv, headers) +// ReplyBlockHeaders creates a reply with a batch of block headers +func (p *peer) ReplyBlockHeaders(reqID uint64, headers []*types.Header) *reply { + data, _ := rlp.EncodeToBytes(headers) + return &reply{p.rw, BlockHeadersMsg, reqID, data} } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// ReplyBlockBodiesRLP creates a reply with a batch of block contents from // an already RLP encoded format. -func (p *peer) SendBlockBodiesRLP(reqID, bv uint64, bodies []rlp.RawValue) error { - return sendResponse(p.rw, BlockBodiesMsg, reqID, bv, bodies) +func (p *peer) ReplyBlockBodiesRLP(reqID uint64, bodies []rlp.RawValue) *reply { + data, _ := rlp.EncodeToBytes(bodies) + return &reply{p.rw, BlockBodiesMsg, reqID, data} } -// SendCodeRLP sends a batch of arbitrary internal data, corresponding to the +// ReplyCode creates a reply with a batch of arbitrary internal data, corresponding to the // hashes requested. -func (p *peer) SendCode(reqID, bv uint64, data [][]byte) error { - return sendResponse(p.rw, CodeMsg, reqID, bv, data) +func (p *peer) ReplyCode(reqID uint64, codes [][]byte) *reply { + data, _ := rlp.EncodeToBytes(codes) + return &reply{p.rw, CodeMsg, reqID, data} } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the +// ReplyReceiptsRLP creates a reply with a batch of transaction receipts, corresponding to the // ones requested from an already RLP encoded format. -func (p *peer) SendReceiptsRLP(reqID, bv uint64, receipts []rlp.RawValue) error { - return sendResponse(p.rw, ReceiptsMsg, reqID, bv, receipts) +func (p *peer) ReplyReceiptsRLP(reqID uint64, receipts []rlp.RawValue) *reply { + data, _ := rlp.EncodeToBytes(receipts) + return &reply{p.rw, ReceiptsMsg, reqID, data} } -// SendProofs sends a batch of legacy LES/1 merkle proofs, corresponding to the ones requested. -func (p *peer) SendProofs(reqID, bv uint64, proofs proofsData) error { - return sendResponse(p.rw, ProofsV1Msg, reqID, bv, proofs) +// ReplyProofs creates a reply with a batch of legacy LES/1 merkle proofs, corresponding to the ones requested. +func (p *peer) ReplyProofs(reqID uint64, proofs proofsData) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, ProofsV1Msg, reqID, data} } -// SendProofsV2 sends a batch of merkle proofs, corresponding to the ones requested. -func (p *peer) SendProofsV2(reqID, bv uint64, proofs light.NodeList) error { - return sendResponse(p.rw, ProofsV2Msg, reqID, bv, proofs) +// ReplyProofsV2 creates a reply with a batch of merkle proofs, corresponding to the ones requested. +func (p *peer) ReplyProofsV2(reqID uint64, proofs light.NodeList) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, ProofsV2Msg, reqID, data} } -// SendHeaderProofs sends a batch of legacy LES/1 header proofs, corresponding to the ones requested. -func (p *peer) SendHeaderProofs(reqID, bv uint64, proofs []ChtResp) error { - return sendResponse(p.rw, HeaderProofsMsg, reqID, bv, proofs) +// ReplyHeaderProofs creates a reply with a batch of legacy LES/1 header proofs, corresponding to the ones requested. +func (p *peer) ReplyHeaderProofs(reqID uint64, proofs []ChtResp) *reply { + data, _ := rlp.EncodeToBytes(proofs) + return &reply{p.rw, HeaderProofsMsg, reqID, data} } -// SendHelperTrieProofs sends a batch of HelperTrie proofs, corresponding to the ones requested. -func (p *peer) SendHelperTrieProofs(reqID, bv uint64, resp HelperTrieResps) error { - return sendResponse(p.rw, HelperTrieProofsMsg, reqID, bv, resp) +// ReplyHelperTrieProofs creates a reply with a batch of HelperTrie proofs, corresponding to the ones requested. +func (p *peer) ReplyHelperTrieProofs(reqID uint64, resp HelperTrieResps) *reply { + data, _ := rlp.EncodeToBytes(resp) + return &reply{p.rw, HelperTrieProofsMsg, reqID, data} } -// SendTxStatus sends a batch of transaction status records, corresponding to the ones requested. -func (p *peer) SendTxStatus(reqID, bv uint64, stats []txStatus) error { - return sendResponse(p.rw, TxStatusMsg, reqID, bv, stats) +// ReplyTxStatus creates a reply with a batch of transaction status records, corresponding to the ones requested. +func (p *peer) ReplyTxStatus(reqID uint64, stats []txStatus) *reply { + data, _ := rlp.EncodeToBytes(stats) + return &reply{p.rw, TxStatusMsg, reqID, data} } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the @@ -311,9 +413,9 @@ func (p *peer) RequestTxStatus(reqID, cost uint64, txHashes []common.Hash) error return sendRequest(p.rw, GetTxStatusMsg, reqID, cost, txHashes) } -// SendTxStatus sends a batch of transactions to be added to the remote transaction pool. -func (p *peer) SendTxs(reqID, cost uint64, txs types.Transactions) error { - p.Log().Debug("Fetching batch of transactions", "count", len(txs)) +// SendTxStatus creates a reply with a batch of transactions to be added to the remote transaction pool. +func (p *peer) SendTxs(reqID, cost uint64, txs rlp.RawValue) error { + p.Log().Debug("Sending batch of transactions", "size", len(txs)) switch p.version { case lpv1: return p2p.Send(p.rw, SendTxMsg, txs) // old message format does not include reqID @@ -344,12 +446,14 @@ func (l keyValueList) add(key string, val interface{}) keyValueList { return append(l, entry) } -func (l keyValueList) decode() keyValueMap { +func (l keyValueList) decode() (keyValueMap, uint64) { m := make(keyValueMap) + var size uint64 for _, entry := range l { m[entry.Key] = entry.Value + size += uint64(len(entry.Key)) + uint64(len(entry.Value)) + 8 } - return m + return m, size } func (m keyValueMap) get(key string, val interface{}) error { @@ -414,9 +518,15 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis } send = send.add("flowControl/BL", server.defParams.BufLimit) send = send.add("flowControl/MRR", server.defParams.MinRecharge) - list := server.fcCostStats.getCurrentList() - send = send.add("flowControl/MRC", list) - p.fcCosts = list.decode() + var costList RequestCostList + if server.costTracker != nil { + costList = server.costTracker.makeCostList() + } else { + costList = testCostList() + } + send = send.add("flowControl/MRC", costList) + p.fcCosts = costList.decode() + p.fcParams = server.defParams } else { //on client node p.announceType = announceTypeSimple @@ -430,8 +540,10 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis if err != nil { return err } - - recv := recvList.decode() + recv, size := recvList.decode() + if p.rejectUpdate(size) { + return errResp(ErrRequestRejected, "") + } var rGenesis, rHash common.Hash var rVersion, rNetwork, rNum uint64 @@ -492,7 +604,7 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis return errResp(ErrUselessPeer, "peer cannot serve requests") } - params := &flowcontrol.ServerParams{} + var params flowcontrol.ServerParams if err := recv.get("flowControl/BL", ¶ms.BufLimit); err != nil { return err } @@ -503,14 +615,38 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis if err := recv.get("flowControl/MRC", &MRC); err != nil { return err } - p.fcServerParams = params - p.fcServer = flowcontrol.NewServerNode(params) + p.fcParams = params + p.fcServer = flowcontrol.NewServerNode(params, &mclock.System{}) p.fcCosts = MRC.decode() } p.headInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum} return nil } +// updateFlowControl updates the flow control parameters belonging to the server +// node if the announced key/value set contains relevant fields +func (p *peer) updateFlowControl(update keyValueMap) { + if p.fcServer == nil { + return + } + params := p.fcParams + updateParams := false + if update.get("flowControl/BL", ¶ms.BufLimit) == nil { + updateParams = true + } + if update.get("flowControl/MRR", ¶ms.MinRecharge) == nil { + updateParams = true + } + if updateParams { + p.fcParams = params + p.fcServer.UpdateParams(params) + } + var MRC RequestCostList + if update.get("flowControl/MRC", &MRC) == nil { + p.fcCosts = MRC.decode() + } +} + // String implements fmt.Stringer. func (p *peer) String() string { return fmt.Sprintf("Peer %s [%s]", p.id, diff --git a/les/peer_test.go b/les/peer_test.go index a0e0a300cb..8b12dd291c 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" @@ -34,7 +35,7 @@ func TestPeerHandshakeSetAnnounceTypeToAnnounceTypeSignedForTrustedPeer(t *testi rw: &rwStub{ WriteHook: func(recvList keyValueList) { //checking that ulc sends to peer allowedRequests=onlyAnnounceRequests and announceType = announceTypeSigned - recv := recvList.decode() + recv, _ := recvList.decode() var reqType uint64 err := recv.get("announceType", &reqType) @@ -79,7 +80,7 @@ func TestPeerHandshakeAnnounceTypeSignedForTrustedPeersPeerNotInTrusted(t *testi rw: &rwStub{ WriteHook: func(recvList keyValueList) { //checking that ulc sends to peer allowedRequests=noRequests and announceType != announceTypeSigned - recv := recvList.decode() + recv, _ := recvList.decode() var reqType uint64 err := recv.get("announceType", &reqType) @@ -237,17 +238,11 @@ func TestPeerHandshakeClientReturnErrorOnUselessPeer(t *testing.T) { func generateLesServer() *LesServer { s := &LesServer{ - defParams: &flowcontrol.ServerParams{ + defParams: flowcontrol.ServerParams{ BufLimit: uint64(300000000), MinRecharge: uint64(50000), }, - fcManager: flowcontrol.NewClientManager(1, 2, 3), - fcCostStats: &requestCostStats{ - stats: make(map[uint64]*linReg, len(reqList)), - }, - } - for _, code := range reqList { - s.fcCostStats.stats[code] = &linReg{cnt: 100} + fcManager: flowcontrol.NewClientManager(nil, &mclock.System{}), } return s } diff --git a/les/protocol.go b/les/protocol.go index b75f92bf74..65395ac052 100644 --- a/les/protocol.go +++ b/les/protocol.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( @@ -81,6 +80,25 @@ const ( TxStatusMsg = 0x15 ) +type requestInfo struct { + name string + maxCount uint64 +} + +var requests = map[uint64]requestInfo{ + GetBlockHeadersMsg: {"GetBlockHeaders", MaxHeaderFetch}, + GetBlockBodiesMsg: {"GetBlockBodies", MaxBodyFetch}, + GetReceiptsMsg: {"GetReceipts", MaxReceiptFetch}, + GetProofsV1Msg: {"GetProofsV1", MaxProofsFetch}, + GetCodeMsg: {"GetCode", MaxCodeFetch}, + SendTxMsg: {"SendTx", MaxTxSend}, + GetHeaderProofsMsg: {"GetHeaderProofs", MaxHelperTrieProofsFetch}, + GetProofsV2Msg: {"GetProofsV2", MaxProofsFetch}, + GetHelperTrieProofsMsg: {"GetHelperTrieProofs", MaxHelperTrieProofsFetch}, + SendTxV2Msg: {"SendTxV2", MaxTxSend}, + GetTxStatusMsg: {"GetTxStatus", MaxTxStatus}, +} + type errCode int const ( @@ -146,9 +164,9 @@ func (a *announceData) sign(privKey *ecdsa.PrivateKey) { } // checkSignature verifies if the block announcement has a valid signature by the given pubKey -func (a *announceData) checkSignature(id enode.ID) error { +func (a *announceData) checkSignature(id enode.ID, update keyValueMap) error { var sig []byte - if err := a.Update.decode().get("sign", &sig); err != nil { + if err := update.get("sign", &sig); err != nil { return err } rlp, _ := rlp.EncodeToBytes(announceBlock{a.Hash, a.Number, a.Td}) diff --git a/les/randselect.go b/les/randselect.go index 1cc1d3d3e0..8efe0c94d3 100644 --- a/les/randselect.go +++ b/les/randselect.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( diff --git a/les/retrieve.go b/les/retrieve.go index d77cfea74e..dd9d14598b 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package les import ( diff --git a/les/server.go b/les/server.go index 2ded3c184b..270640f025 100644 --- a/les/server.go +++ b/les/server.go @@ -14,40 +14,46 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( "crypto/ecdsa" - "encoding/binary" - "math" "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discv5" "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" ) +const bufLimitRatio = 6000 // fixed bufLimit/MRR ratio + type LesServer struct { lesCommons fcManager *flowcontrol.ClientManager // nil if our node is client only - fcCostStats *requestCostStats - defParams *flowcontrol.ServerParams + costTracker *costTracker + defParams flowcontrol.ServerParams lesTopics []discv5.Topic privateKey *ecdsa.PrivateKey quitSync chan struct{} onlyAnnounce bool + + thcNormal, thcBlockProcessing int // serving thread count for normal operation and block processing mode + + maxPeers int + freeClientCap uint64 + freeClientPool *freeClientPool + priorityClientPool *priorityClientPool } func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { @@ -87,12 +93,20 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { bloomTrieIndexer: light.NewBloomTrieIndexer(eth.ChainDb(), nil, params.BloomBitsBlocks, params.BloomTrieFrequency), protocolManager: pm, }, + costTracker: newCostTracker(eth.ChainDb(), config), quitSync: quitSync, lesTopics: lesTopics, onlyAnnounce: config.OnlyAnnounce, } logger := log.New() + pm.server = srv + srv.thcNormal = config.LightServ * 4 / 100 + if srv.thcNormal < 4 { + srv.thcNormal = 4 + } + srv.thcBlockProcessing = config.LightServ/100 + 1 + srv.fcManager = flowcontrol.NewClientManager(nil, &mclock.System{}) chtV1SectionCount, _, _ := srv.chtIndexer.Sections() // indexer still uses LES/1 4k section size for backwards server compatibility chtV2SectionCount := chtV1SectionCount / (params.CHTFrequencyClient / params.CHTFrequencyServer) @@ -114,23 +128,92 @@ func NewLesServer(eth *eth.Ethereum, config *eth.Config) (*LesServer, error) { } srv.chtIndexer.Start(eth.BlockChain()) - pm.server = srv - - srv.defParams = &flowcontrol.ServerParams{ - BufLimit: 300000000, - MinRecharge: 50000, - } - srv.fcManager = flowcontrol.NewClientManager(uint64(config.LightServ), 10, 1000000000) - srv.fcCostStats = newCostStats(eth.ChainDb()) return srv, nil } +func (s *LesServer) APIs() []rpc.API { + return []rpc.API{ + { + Namespace: "les", + Version: "1.0", + Service: NewPrivateLightServerAPI(s), + Public: false, + }, + } +} + +// startEventLoop starts an event handler loop that updates the recharge curve of +// the client manager and adjusts the client pool's size according to the total +// capacity updates coming from the client manager +func (s *LesServer) startEventLoop() { + s.protocolManager.wg.Add(1) + + var processing bool + blockProcFeed := make(chan bool, 100) + s.protocolManager.blockchain.(*core.BlockChain).SubscribeBlockProcessingEvent(blockProcFeed) + totalRechargeCh := make(chan uint64, 100) + totalRecharge := s.costTracker.subscribeTotalRecharge(totalRechargeCh) + totalCapacityCh := make(chan uint64, 100) + updateRecharge := func() { + if processing { + s.protocolManager.servingQueue.setThreads(s.thcBlockProcessing) + s.fcManager.SetRechargeCurve(flowcontrol.PieceWiseLinear{{0, 0}, {totalRecharge, totalRecharge}}) + } else { + s.protocolManager.servingQueue.setThreads(s.thcNormal) + s.fcManager.SetRechargeCurve(flowcontrol.PieceWiseLinear{{0, 0}, {totalRecharge / 10, totalRecharge}, {totalRecharge, totalRecharge}}) + } + } + updateRecharge() + totalCapacity := s.fcManager.SubscribeTotalCapacity(totalCapacityCh) + s.priorityClientPool.setLimits(s.maxPeers, totalCapacity) + + go func() { + for { + select { + case processing = <-blockProcFeed: + updateRecharge() + case totalRecharge = <-totalRechargeCh: + updateRecharge() + case totalCapacity = <-totalCapacityCh: + s.priorityClientPool.setLimits(s.maxPeers, totalCapacity) + case <-s.protocolManager.quitSync: + s.protocolManager.wg.Done() + return + } + } + }() +} + func (s *LesServer) Protocols() []p2p.Protocol { return s.makeProtocols(ServerProtocolVersions) } // Start starts the LES server func (s *LesServer) Start(srvr *p2p.Server) { + s.maxPeers = s.config.LightPeers + totalRecharge := s.costTracker.totalRecharge() + if s.maxPeers > 0 { + s.freeClientCap = minCapacity //totalRecharge / uint64(s.maxPeers) + if s.freeClientCap < minCapacity { + s.freeClientCap = minCapacity + } + if s.freeClientCap > 0 { + s.defParams = flowcontrol.ServerParams{ + BufLimit: s.freeClientCap * bufLimitRatio, + MinRecharge: s.freeClientCap, + } + } + } + freePeers := int(totalRecharge / s.freeClientCap) + if freePeers < s.maxPeers { + log.Warn("Light peer count limited", "specified", s.maxPeers, "allowed", freePeers) + } + + s.freeClientPool = newFreeClientPool(s.chainDb, s.freeClientCap, 10000, mclock.System{}, func(id string) { go s.protocolManager.removePeer(id) }) + s.priorityClientPool = newPriorityClientPool(s.freeClientCap, s.protocolManager.peers, s.freeClientPool) + + s.protocolManager.peers.notify(s.priorityClientPool) + s.startEventLoop() s.protocolManager.Start(s.config.LightPeers) if srvr.DiscV5 != nil { for _, topic := range s.lesTopics { @@ -156,185 +239,14 @@ func (s *LesServer) SetBloomBitsIndexer(bloomIndexer *core.ChainIndexer) { func (s *LesServer) Stop() { s.chtIndexer.Close() // bloom trie indexer is closed by parent bloombits indexer - s.fcCostStats.store() - s.fcManager.Stop() go func() { <-s.protocolManager.noMorePeers }() + s.freeClientPool.stop() + s.costTracker.stop() s.protocolManager.Stop() } -type requestCosts struct { - baseCost, reqCost uint64 -} - -type requestCostTable map[uint64]*requestCosts - -type RequestCostList []struct { - MsgCode, BaseCost, ReqCost uint64 -} - -func (list RequestCostList) decode() requestCostTable { - table := make(requestCostTable) - for _, e := range list { - table[e.MsgCode] = &requestCosts{ - baseCost: e.BaseCost, - reqCost: e.ReqCost, - } - } - return table -} - -type linReg struct { - sumX, sumY, sumXX, sumXY float64 - cnt uint64 -} - -const linRegMaxCnt = 100000 - -func (l *linReg) add(x, y float64) { - if l.cnt >= linRegMaxCnt { - sub := float64(l.cnt+1-linRegMaxCnt) / linRegMaxCnt - l.sumX -= l.sumX * sub - l.sumY -= l.sumY * sub - l.sumXX -= l.sumXX * sub - l.sumXY -= l.sumXY * sub - l.cnt = linRegMaxCnt - 1 - } - l.cnt++ - l.sumX += x - l.sumY += y - l.sumXX += x * x - l.sumXY += x * y -} - -func (l *linReg) calc() (b, m float64) { - if l.cnt == 0 { - return 0, 0 - } - cnt := float64(l.cnt) - d := cnt*l.sumXX - l.sumX*l.sumX - if d < 0.001 { - return l.sumY / cnt, 0 - } - m = (cnt*l.sumXY - l.sumX*l.sumY) / d - b = (l.sumY / cnt) - (m * l.sumX / cnt) - return b, m -} - -func (l *linReg) toBytes() []byte { - var arr [40]byte - binary.BigEndian.PutUint64(arr[0:8], math.Float64bits(l.sumX)) - binary.BigEndian.PutUint64(arr[8:16], math.Float64bits(l.sumY)) - binary.BigEndian.PutUint64(arr[16:24], math.Float64bits(l.sumXX)) - binary.BigEndian.PutUint64(arr[24:32], math.Float64bits(l.sumXY)) - binary.BigEndian.PutUint64(arr[32:40], l.cnt) - return arr[:] -} - -func linRegFromBytes(data []byte) *linReg { - if len(data) != 40 { - return nil - } - l := &linReg{} - l.sumX = math.Float64frombits(binary.BigEndian.Uint64(data[0:8])) - l.sumY = math.Float64frombits(binary.BigEndian.Uint64(data[8:16])) - l.sumXX = math.Float64frombits(binary.BigEndian.Uint64(data[16:24])) - l.sumXY = math.Float64frombits(binary.BigEndian.Uint64(data[24:32])) - l.cnt = binary.BigEndian.Uint64(data[32:40]) - return l -} - -type requestCostStats struct { - lock sync.RWMutex - db ethdb.Database - stats map[uint64]*linReg -} - -type requestCostStatsRlp []struct { - MsgCode uint64 - Data []byte -} - -var rcStatsKey = []byte("_requestCostStats") - -func newCostStats(db ethdb.Database) *requestCostStats { - stats := make(map[uint64]*linReg) - for _, code := range reqList { - stats[code] = &linReg{cnt: 100} - } - - if db != nil { - data, err := db.Get(rcStatsKey) - var statsRlp requestCostStatsRlp - if err == nil { - err = rlp.DecodeBytes(data, &statsRlp) - } - if err == nil { - for _, r := range statsRlp { - if stats[r.MsgCode] != nil { - if l := linRegFromBytes(r.Data); l != nil { - stats[r.MsgCode] = l - } - } - } - } - } - - return &requestCostStats{ - db: db, - stats: stats, - } -} - -func (s *requestCostStats) store() { - s.lock.Lock() - defer s.lock.Unlock() - - statsRlp := make(requestCostStatsRlp, len(reqList)) - for i, code := range reqList { - statsRlp[i].MsgCode = code - statsRlp[i].Data = s.stats[code].toBytes() - } - - if data, err := rlp.EncodeToBytes(statsRlp); err == nil { - s.db.Put(rcStatsKey, data) - } -} - -func (s *requestCostStats) getCurrentList() RequestCostList { - s.lock.Lock() - defer s.lock.Unlock() - - list := make(RequestCostList, len(reqList)) - for idx, code := range reqList { - b, m := s.stats[code].calc() - if m < 0 { - b += m - m = 0 - } - if b < 0 { - b = 0 - } - - list[idx].MsgCode = code - list[idx].BaseCost = uint64(b * 2) - list[idx].ReqCost = uint64(m * 2) - } - return list -} - -func (s *requestCostStats) update(msgCode, reqCnt, cost uint64) { - s.lock.Lock() - defer s.lock.Unlock() - - c, ok := s.stats[msgCode] - if !ok || reqCnt == 0 { - return - } - c.add(float64(reqCnt), float64(cost)) -} - func (pm *ProtocolManager) blockLoop() { pm.wg.Add(1) headCh := make(chan core.ChainHeadEvent, 10) @@ -371,12 +283,7 @@ func (pm *ProtocolManager) blockLoop() { switch p.announceType { case announceTypeSimple: - select { - case p.announceChn <- announce: - default: - pm.removePeer(p.id) - } - + p.queueSend(func() { p.SendAnnounce(announce) }) case announceTypeSigned: if !signed { signedAnnounce = announce @@ -384,11 +291,7 @@ func (pm *ProtocolManager) blockLoop() { signed = true } - select { - case p.announceChn <- signedAnnounce: - default: - pm.removePeer(p.id) - } + p.queueSend(func() { p.SendAnnounce(signedAnnounce) }) } } } diff --git a/les/serverpool.go b/les/serverpool.go index 3f4d0a1d9b..668f39c562 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -14,7 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package les implements the Light Ethereum Subprotocol. package les import ( diff --git a/les/servingqueue.go b/les/servingqueue.go new file mode 100644 index 0000000000..2438fdfe3c --- /dev/null +++ b/les/servingqueue.go @@ -0,0 +1,261 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package les + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/common/prque" +) + +// servingQueue allows running tasks in a limited number of threads and puts the +// waiting tasks in a priority queue +type servingQueue struct { + tokenCh chan runToken + queueAddCh, queueBestCh chan *servingTask + stopThreadCh, quit chan struct{} + setThreadsCh chan int + + wg sync.WaitGroup + threadCount int // number of currently running threads + queue *prque.Prque // priority queue for waiting or suspended tasks + best *servingTask // the highest priority task (not included in the queue) + suspendBias int64 // priority bias against suspending an already running task +} + +// servingTask represents a request serving task. Tasks can be implemented to +// run in multiple steps, allowing the serving queue to suspend execution between +// steps if higher priority tasks are entered. The creator of the task should +// set the following fields: +// +// - priority: greater value means higher priority; values can wrap around the int64 range +// - run: execute a single step; return true if finished +// - after: executed after run finishes or returns an error, receives the total serving time +type servingTask struct { + sq *servingQueue + servingTime uint64 + priority int64 + biasAdded bool + token runToken + tokenCh chan runToken +} + +// runToken received by servingTask.start allows the task to run. Closing the +// channel by servingTask.stop signals the thread controller to allow a new task +// to start running. +type runToken chan struct{} + +// start blocks until the task can start and returns true if it is allowed to run. +// Returning false means that the task should be cancelled. +func (t *servingTask) start() bool { + select { + case t.token = <-t.sq.tokenCh: + default: + t.tokenCh = make(chan runToken, 1) + select { + case t.sq.queueAddCh <- t: + case <-t.sq.quit: + return false + } + select { + case t.token = <-t.tokenCh: + case <-t.sq.quit: + return false + } + } + if t.token == nil { + return false + } + t.servingTime -= uint64(mclock.Now()) + return true +} + +// done signals the thread controller about the task being finished and returns +// the total serving time of the task in nanoseconds. +func (t *servingTask) done() uint64 { + t.servingTime += uint64(mclock.Now()) + close(t.token) + return t.servingTime +} + +// waitOrStop can be called during the execution of the task. It blocks if there +// is a higher priority task waiting (a bias is applied in favor of the currently +// running task). Returning true means that the execution can be resumed. False +// means the task should be cancelled. +func (t *servingTask) waitOrStop() bool { + t.done() + if !t.biasAdded { + t.priority += t.sq.suspendBias + t.biasAdded = true + } + return t.start() +} + +// newServingQueue returns a new servingQueue +func newServingQueue(suspendBias int64) *servingQueue { + sq := &servingQueue{ + queue: prque.New(nil), + suspendBias: suspendBias, + tokenCh: make(chan runToken), + queueAddCh: make(chan *servingTask, 100), + queueBestCh: make(chan *servingTask), + stopThreadCh: make(chan struct{}), + quit: make(chan struct{}), + setThreadsCh: make(chan int, 10), + } + sq.wg.Add(2) + go sq.queueLoop() + go sq.threadCountLoop() + return sq +} + +// newTask creates a new task with the given priority +func (sq *servingQueue) newTask(priority int64) *servingTask { + return &servingTask{ + sq: sq, + priority: priority, + } +} + +// threadController is started in multiple goroutines and controls the execution +// of tasks. The number of active thread controllers equals the allowed number of +// concurrently running threads. It tries to fetch the highest priority queued +// task first. If there are no queued tasks waiting then it can directly catch +// run tokens from the token channel and allow the corresponding tasks to run +// without entering the priority queue. +func (sq *servingQueue) threadController() { + for { + token := make(runToken) + select { + case best := <-sq.queueBestCh: + best.tokenCh <- token + default: + select { + case best := <-sq.queueBestCh: + best.tokenCh <- token + case sq.tokenCh <- token: + case <-sq.stopThreadCh: + sq.wg.Done() + return + case <-sq.quit: + sq.wg.Done() + return + } + } + <-token + select { + case <-sq.stopThreadCh: + sq.wg.Done() + return + case <-sq.quit: + sq.wg.Done() + return + default: + } + } +} + +// addTask inserts a task into the priority queue +func (sq *servingQueue) addTask(task *servingTask) { + if sq.best == nil { + sq.best = task + } else if task.priority > sq.best.priority { + sq.queue.Push(sq.best, sq.best.priority) + sq.best = task + return + } else { + sq.queue.Push(task, task.priority) + } +} + +// queueLoop is an event loop running in a goroutine. It receives tasks from queueAddCh +// and always tries to send the highest priority task to queueBestCh. Successfully sent +// tasks are removed from the queue. +func (sq *servingQueue) queueLoop() { + for { + if sq.best != nil { + select { + case task := <-sq.queueAddCh: + sq.addTask(task) + case sq.queueBestCh <- sq.best: + if sq.queue.Size() == 0 { + sq.best = nil + } else { + sq.best, _ = sq.queue.PopItem().(*servingTask) + } + case <-sq.quit: + sq.wg.Done() + return + } + } else { + select { + case task := <-sq.queueAddCh: + sq.addTask(task) + case <-sq.quit: + sq.wg.Done() + return + } + } + } +} + +// threadCountLoop is an event loop running in a goroutine. It adjusts the number +// of active thread controller goroutines. +func (sq *servingQueue) threadCountLoop() { + var threadCountTarget int + for { + for threadCountTarget > sq.threadCount { + sq.wg.Add(1) + go sq.threadController() + sq.threadCount++ + } + if threadCountTarget < sq.threadCount { + select { + case threadCountTarget = <-sq.setThreadsCh: + case sq.stopThreadCh <- struct{}{}: + sq.threadCount-- + case <-sq.quit: + sq.wg.Done() + return + } + } else { + select { + case threadCountTarget = <-sq.setThreadsCh: + case <-sq.quit: + sq.wg.Done() + return + } + } + } +} + +// setThreads sets the allowed processing thread count, suspending tasks as soon as +// possible if necessary. +func (sq *servingQueue) setThreads(threadCount int) { + select { + case sq.setThreadsCh <- threadCount: + case <-sq.quit: + return + } +} + +// stop stops task processing as soon as possible and shuts down the serving queue. +func (sq *servingQueue) stop() { + close(sq.quit) + sq.wg.Wait() +} diff --git a/les/txrelay.go b/les/txrelay.go index 6d22856f9f..a790bbec9c 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" ) type ltrInfo struct { @@ -113,21 +114,22 @@ func (self *LesTxRelay) send(txs types.Transactions, count int) { for p, list := range sendTo { pp := p ll := list + enc, _ := rlp.EncodeToBytes(ll) reqID := genReqID() rq := &distReq{ getCost: func(dp distPeer) uint64 { peer := dp.(*peer) - return peer.GetRequestCost(SendTxMsg, len(ll)) + return peer.GetTxRelayCost(len(ll), len(enc)) }, canSend: func(dp distPeer) bool { return !dp.(*peer).isOnlyAnnounce && dp.(*peer) == pp }, request: func(dp distPeer) func() { peer := dp.(*peer) - cost := peer.GetRequestCost(SendTxMsg, len(ll)) - peer.fcServer.QueueRequest(reqID, cost) - return func() { peer.SendTxs(reqID, cost, ll) } + cost := peer.GetTxRelayCost(len(ll), len(enc)) + peer.fcServer.QueuedRequest(reqID, cost) + return func() { peer.SendTxs(reqID, cost, enc) } }, } self.reqDist.queue(rq) diff --git a/les/ulc_test.go b/les/ulc_test.go index 3b95e6368e..69ea62059d 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -11,6 +11,7 @@ import ( "crypto/ecdsa" "math/big" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -217,7 +218,7 @@ func newFullPeerPair(t *testing.T, index int, numberOfblocks int, chainGen func( // newLightPeer creates node with light sync mode func newLightPeer(t *testing.T, ulcConfig *eth.ULCConfig) pairPeer { peers := newPeerSet() - dist := newRequestDistributor(peers, make(chan struct{})) + dist := newRequestDistributor(peers, make(chan struct{}), &mclock.System{}) rm := newRetrieveManager(peers, dist, nil) ldb := ethdb.NewMemDatabase() diff --git a/light/lightchain.go b/light/lightchain.go index 5019622c79..fb5f8ead25 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// Package light implements on-demand retrieval capable state and chain objects +// for the Ethereum Light Client. package light import ( diff --git a/light/odr.go b/light/odr.go index 900be05440..95f1948e75 100644 --- a/light/odr.go +++ b/light/odr.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. package light import ( diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 9b588db1be..bd8bcbc856 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -97,7 +97,11 @@ func (e *ExecAdapter) NewNode(config *NodeConfig) (Node, error) { Stack: node.DefaultConfig, Node: config, } - conf.Stack.DataDir = filepath.Join(dir, "data") + if config.DataDir != "" { + conf.Stack.DataDir = config.DataDir + } else { + conf.Stack.DataDir = filepath.Join(dir, "data") + } conf.Stack.WSHost = "127.0.0.1" conf.Stack.WSPort = 0 conf.Stack.WSOrigins = []string{"*"} diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 6681726e40..31856b76dd 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -90,6 +90,9 @@ type NodeConfig struct { // Name is a human friendly name for the node like "node01" Name string + // Use an existing database instead of a temporary one if non-empty + DataDir string + // Services are the names of the services which should be run when // starting the node (for SimNodes it should be the names of services // contained in SimAdapter.services, for other nodes it should be From b7e0dec6bdf40c94d51740fd2cf13321f69a6b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 26 Feb 2019 16:56:12 +0200 Subject: [PATCH 17/21] travis, build: switch to NDK 19b, fix gomobile builds (#19171) * travis, build: switch to NDK 19b, fix gomobile builds * travis, build: move NDK into its final bundle location * travis: disable Android build on PRs once again --- .travis.yml | 7 +++---- build/ci.go | 4 ---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5d61391138..b09bd7ce04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -165,10 +165,9 @@ matrix: - export GOPATH=$HOME/go script: # Build the Android archive and upload it to Maven Central and Azure - - curl https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip -o android-ndk-r17b.zip - - unzip -q android-ndk-r17b.zip && rm android-ndk-r17b.zip - - mv android-ndk-r17b $HOME - - export ANDROID_NDK=$HOME/android-ndk-r17b + - curl https://dl.google.com/android/repository/android-ndk-r19b-linux-x86_64.zip -o android-ndk-r19b.zip + - unzip -q android-ndk-r19b.zip && rm android-ndk-r19b.zip + - mv android-ndk-r19b $ANDROID_HOME/ndk-bundle - mkdir -p $GOPATH/src/github.com/ethereum - ln -s `pwd` $GOPATH/src/github.com/ethereum/go-ethereum diff --git a/build/ci.go b/build/ci.go index 4ee76ced51..91339d22c8 100644 --- a/build/ci.go +++ b/build/ci.go @@ -800,12 +800,8 @@ func doAndroidArchive(cmdline []string) { if os.Getenv("ANDROID_HOME") == "" { log.Fatal("Please ensure ANDROID_HOME points to your Android SDK") } - if os.Getenv("ANDROID_NDK") == "" { - log.Fatal("Please ensure ANDROID_NDK points to your Android NDK") - } // Build the Android archive and Maven resources build.MustRun(goTool("get", "golang.org/x/mobile/cmd/gomobile", "golang.org/x/mobile/cmd/gobind")) - build.MustRun(gomobileTool("init", "--ndk", os.Getenv("ANDROID_NDK"))) build.MustRun(gomobileTool("bind", "-ldflags", "-s -w", "--target", "android", "--javapkg", "org.ethereum", "-v", "github.com/ethereum/go-ethereum/mobile")) if *local { From f0233948d2bc343deb17be06c1a0f93c3e004084 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Tue, 26 Feb 2019 16:09:32 +0100 Subject: [PATCH 18/21] swarm/chunk: move chunk related declarations to chunk package (#19170) --- swarm/chunk/chunk.go | 108 +++++++++++++++- .../types_test.go => chunk/proximity_test.go} | 2 +- swarm/storage/chunker.go | 6 +- swarm/storage/common_test.go | 4 +- swarm/storage/error.go | 9 +- swarm/storage/hasherstore.go | 16 +-- swarm/storage/ldbstore.go | 14 +- swarm/storage/ldbstore_test.go | 8 +- swarm/storage/localstore.go | 2 +- swarm/storage/localstore/gc_test.go | 47 +++++-- swarm/storage/localstore/index_test.go | 10 +- swarm/storage/localstore/localstore.go | 12 +- swarm/storage/localstore/localstore_test.go | 57 ++++----- swarm/storage/localstore/mode_get.go | 12 +- swarm/storage/localstore/mode_get_test.go | 4 +- swarm/storage/localstore/mode_put.go | 4 +- swarm/storage/localstore/mode_put_test.go | 18 +-- swarm/storage/localstore/mode_set.go | 6 +- swarm/storage/localstore/mode_set_test.go | 6 +- .../localstore/retrieval_index_test.go | 10 +- swarm/storage/localstore/subscription_pull.go | 4 +- .../localstore/subscription_pull_test.go | 72 +++++------ swarm/storage/localstore/subscription_push.go | 8 +- .../localstore/subscription_push_test.go | 10 +- swarm/storage/localstore_test.go | 12 +- swarm/storage/netstore_test.go | 24 ++-- swarm/storage/pyramid.go | 6 +- swarm/storage/types.go | 120 +++--------------- 28 files changed, 325 insertions(+), 286 deletions(-) rename swarm/{storage/types_test.go => chunk/proximity_test.go} (99%) diff --git a/swarm/chunk/chunk.go b/swarm/chunk/chunk.go index 1449efccd0..7540af8ce9 100644 --- a/swarm/chunk/chunk.go +++ b/swarm/chunk/chunk.go @@ -1,5 +1,109 @@ package chunk -const ( - DefaultSize = 4096 +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" ) + +const ( + DefaultSize = 4096 + MaxPO = 16 + AddressLength = 32 +) + +var ( + ErrChunkNotFound = errors.New("chunk not found") + ErrChunkInvalid = errors.New("invalid chunk") +) + +type Chunk interface { + Address() Address + Data() []byte +} + +type chunk struct { + addr Address + sdata []byte +} + +func NewChunk(addr Address, data []byte) *chunk { + return &chunk{ + addr: addr, + sdata: data, + } +} + +func (c *chunk) Address() Address { + return c.addr +} + +func (c *chunk) Data() []byte { + return c.sdata +} + +func (self *chunk) String() string { + return fmt.Sprintf("Address: %v Chunksize: %v", self.addr.Log(), len(self.sdata)) +} + +type Address []byte + +var ZeroAddr = Address(common.Hash{}.Bytes()) + +func (a Address) Hex() string { + return fmt.Sprintf("%064x", []byte(a[:])) +} + +func (a Address) Log() string { + if len(a[:]) < 8 { + return fmt.Sprintf("%x", []byte(a[:])) + } + return fmt.Sprintf("%016x", []byte(a[:8])) +} + +func (a Address) String() string { + return fmt.Sprintf("%064x", []byte(a)) +} + +func (a Address) MarshalJSON() (out []byte, err error) { + return []byte(`"` + a.String() + `"`), nil +} + +func (a *Address) UnmarshalJSON(value []byte) error { + s := string(value) + *a = make([]byte, 32) + h := common.Hex2Bytes(s[1 : len(s)-1]) + copy(*a, h) + return nil +} + +// Proximity returns the proximity order of the MSB distance between x and y +// +// The distance metric MSB(x, y) of two equal length byte sequences x an y is the +// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. +// the binary cast is big endian: most significant bit first (=MSB). +// +// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. +// It is defined as the reverse rank of the integer part of the base 2 +// logarithm of the distance. +// It is calculated by counting the number of common leading zeros in the (MSB) +// binary representation of the x^y. +// +// (0 farthest, 255 closest, 256 self) +func Proximity(one, other []byte) (ret int) { + b := (MaxPO-1)/8 + 1 + if b > len(one) { + b = len(one) + } + m := 8 + for i := 0; i < b; i++ { + oxo := one[i] ^ other[i] + for j := 0; j < m; j++ { + if (oxo>>uint8(7-j))&0x01 != 0 { + return i*8 + j + } + } + } + return MaxPO +} diff --git a/swarm/storage/types_test.go b/swarm/chunk/proximity_test.go similarity index 99% rename from swarm/storage/types_test.go rename to swarm/chunk/proximity_test.go index 32907bbf49..5632114b16 100644 --- a/swarm/storage/types_test.go +++ b/swarm/chunk/proximity_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package storage +package chunk import ( "strconv" diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 0fa5026dcd..5b36b477e0 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -25,7 +25,7 @@ import ( "time" "github.com/ethereum/go-ethereum/metrics" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/spancontext" opentracing "github.com/opentracing/opentracing-go" @@ -127,7 +127,7 @@ type TreeChunker struct { func TreeJoin(ctx context.Context, addr Address, getter Getter, depth int) *LazyChunkReader { jp := &JoinerParams{ ChunkerParams: ChunkerParams{ - chunkSize: ch.DefaultSize, + chunkSize: chunk.DefaultSize, hashSize: int64(len(addr)), }, addr: addr, @@ -147,7 +147,7 @@ func TreeSplit(ctx context.Context, data io.Reader, size int64, putter Putter) ( tsp := &TreeSplitterParams{ SplitterParams: SplitterParams{ ChunkerParams: ChunkerParams{ - chunkSize: ch.DefaultSize, + chunkSize: chunk.DefaultSize, hashSize: putter.RefSize(), }, reader: data, diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index e74d0f4b84..c4d187b622 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -29,7 +29,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/mattn/go-colorable" ) @@ -94,7 +94,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() for i := int64(0); i < int64(n); i++ { - chunk := f(ch.DefaultSize) + chunk := f(chunk.DefaultSize) go func() { select { case errc <- store.Put(ctx, chunk): diff --git a/swarm/storage/error.go b/swarm/storage/error.go index a9d0616fad..1e412e55ce 100644 --- a/swarm/storage/error.go +++ b/swarm/storage/error.go @@ -16,9 +16,7 @@ package storage -import ( - "errors" -) +import "github.com/ethereum/go-ethereum/swarm/chunk" const ( ErrInit = iota @@ -31,7 +29,8 @@ const ( ErrNotSynced ) +// Errors are the same as the ones in chunk package for backward compatibility. var ( - ErrChunkNotFound = errors.New("chunk not found") - ErrChunkInvalid = errors.New("invalid chunk") + ErrChunkNotFound = chunk.ErrChunkNotFound + ErrChunkInvalid = chunk.ErrChunkNotFound ) diff --git a/swarm/storage/hasherstore.go b/swarm/storage/hasherstore.go index 23b52ee0d8..345ce7430a 100644 --- a/swarm/storage/hasherstore.go +++ b/swarm/storage/hasherstore.go @@ -21,7 +21,7 @@ import ( "fmt" "sync/atomic" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/storage/encryption" "golang.org/x/crypto/sha3" ) @@ -156,7 +156,7 @@ func (h *hasherStore) createHash(chunkData ChunkData) Address { return hasher.Sum(nil) } -func (h *hasherStore) createChunk(chunkData ChunkData) *chunk { +func (h *hasherStore) createChunk(chunkData ChunkData) Chunk { hash := h.createHash(chunkData) chunk := NewChunk(hash, chunkData) return chunk @@ -189,9 +189,9 @@ func (h *hasherStore) decryptChunkData(chunkData ChunkData, encryptionKey encryp // removing extra bytes which were just added for padding length := ChunkData(decryptedSpan).Size() - for length > ch.DefaultSize { - length = length + (ch.DefaultSize - 1) - length = length / ch.DefaultSize + for length > chunk.DefaultSize { + length = length + (chunk.DefaultSize - 1) + length = length / chunk.DefaultSize length *= uint64(h.refSize) } @@ -232,14 +232,14 @@ func (h *hasherStore) decrypt(chunkData ChunkData, key encryption.Key) ([]byte, } func (h *hasherStore) newSpanEncryption(key encryption.Key) encryption.Encryption { - return encryption.New(key, 0, uint32(ch.DefaultSize/h.refSize), sha3.NewLegacyKeccak256) + return encryption.New(key, 0, uint32(chunk.DefaultSize/h.refSize), sha3.NewLegacyKeccak256) } func (h *hasherStore) newDataEncryption(key encryption.Key) encryption.Encryption { - return encryption.New(key, int(ch.DefaultSize), 0, sha3.NewLegacyKeccak256) + return encryption.New(key, int(chunk.DefaultSize), 0, sha3.NewLegacyKeccak256) } -func (h *hasherStore) storeChunk(ctx context.Context, chunk *chunk) { +func (h *hasherStore) storeChunk(ctx context.Context, chunk Chunk) { atomic.AddUint64(&h.nrChunks, 1) go func() { select { diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 1d53577130..766a9e031b 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -312,7 +312,7 @@ func decodeIndex(data []byte, index *dpaDBIndex) error { return dec.Decode(index) } -func decodeData(addr Address, data []byte) (*chunk, error) { +func decodeData(addr Address, data []byte) (Chunk, error) { return NewChunk(addr, data[32:]), nil } @@ -502,7 +502,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) { } // Cleanup iterates over the database and deletes chunks if they pass the `f` condition -func (s *LDBStore) Cleanup(f func(*chunk) bool) { +func (s *LDBStore) Cleanup(f func(Chunk) bool) { var errorsFound, removed, total int it := s.db.NewIterator() @@ -551,12 +551,14 @@ func (s *LDBStore) Cleanup(f func(*chunk) bool) { continue } - cs := int64(binary.LittleEndian.Uint64(c.sdata[:8])) - log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) + sdata := c.Data() + + cs := int64(binary.LittleEndian.Uint64(sdata[:8])) + log.Trace("chunk", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs) // if chunk is to be removed if f(c) { - log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(c.sdata), "size", cs) + log.Warn("chunk for cleanup", "key", fmt.Sprintf("%x", key), "ck", fmt.Sprintf("%x", ck), "dkey", fmt.Sprintf("%x", datakey), "dataidx", index.Idx, "po", po, "len data", len(data), "len sdata", len(sdata), "size", cs) s.deleteNow(&index, getIndexKey(key[1:]), po) removed++ errorsFound++ @@ -980,7 +982,7 @@ func (s *LDBStore) Has(_ context.Context, addr Address) bool { } // TODO: To conform with other private methods of this object indices should not be updated -func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { +func (s *LDBStore) get(addr Address) (chunk Chunk, err error) { if s.closed { return nil, ErrDBClosed } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index aa65183e33..d17bd7d0e4 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -28,7 +28,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/common" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" ldberrors "github.com/syndtr/goleveldb/leveldb/errors" @@ -103,7 +103,7 @@ func TestMarkAccessed(t *testing.T) { t.Fatalf("init dbStore failed: %v", err) } - h := GenerateRandomChunk(ch.DefaultSize) + h := GenerateRandomChunk(chunk.DefaultSize) db.Put(context.Background(), h) @@ -201,7 +201,7 @@ func testIterator(t *testing.T, mock bool) { t.Fatalf("init dbStore failed: %v", err) } - chunks := GenerateRandomChunks(ch.DefaultSize, chunkcount) + chunks := GenerateRandomChunks(chunk.DefaultSize, chunkcount) for i = 0; i < len(chunks); i++ { chunkkeys[i] = chunks[i].Address() @@ -468,7 +468,7 @@ func testLDBStoreRemoveThenCollectGarbage(t *testing.T) { // put capacity count number of chunks chunks := make([]Chunk, n) for i := 0; i < n; i++ { - c := GenerateRandomChunk(ch.DefaultSize) + c := GenerateRandomChunk(chunk.DefaultSize) chunks[i] = c log.Trace("generate random chunk", "idx", i, "chunk", c) } diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index eefb7565a5..a8f6f037f6 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -241,7 +241,7 @@ func (ls *LocalStore) Migrate() error { func (ls *LocalStore) migrateFromNoneToPurity() { // delete chunks that are not valid, i.e. chunks that do not pass // any of the ls.Validators - ls.DbStore.Cleanup(func(c *chunk) bool { + ls.DbStore.Cleanup(func(c Chunk) bool { return !ls.isValid(c) }) } diff --git a/swarm/storage/localstore/gc_test.go b/swarm/storage/localstore/gc_test.go index 60309d7fa9..3964c16d59 100644 --- a/swarm/storage/localstore/gc_test.go +++ b/swarm/storage/localstore/gc_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // TestDB_collectGarbageWorker tests garbage collection runs @@ -64,11 +64,11 @@ func testDB_collectGarbageWorker(t *testing.T) { uploader := db.NewPutter(ModePutUpload) syncer := db.NewSetter(ModeSetSync) - addrs := make([]storage.Address, 0) + addrs := make([]chunk.Address, 0) // upload random chunks for i := 0; i < chunkCount; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { @@ -106,8 +106,8 @@ func testDB_collectGarbageWorker(t *testing.T) { // the first synced chunk should be removed t.Run("get the first synced chunk", func(t *testing.T) { _, err := db.NewGetter(ModeGetRequest).Get(addrs[0]) - if err != storage.ErrChunkNotFound { - t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound) + if err != chunk.ErrChunkNotFound { + t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound) } }) @@ -137,11 +137,11 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) { testHookCollectGarbageChan <- collectedCount })() - addrs := make([]storage.Address, 0) + addrs := make([]chunk.Address, 0) // upload random chunks just up to the capacity for i := 0; i < int(db.capacity)-1; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { @@ -156,6 +156,14 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) { addrs = append(addrs, chunk.Address()) } + // set update gc test hook to signal when + // update gc goroutine is done by closing + // testHookUpdateGCChan channel + testHookUpdateGCChan := make(chan struct{}) + resetTestHookUpdateGC := setTestHookUpdateGC(func() { + close(testHookUpdateGCChan) + }) + // request the latest synced chunk // to prioritize it in the gc index // not to be collected @@ -164,18 +172,29 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) { t.Fatal(err) } + // wait for update gc goroutine to finish for garbage + // collector to be correctly triggered after the last upload + select { + case <-testHookUpdateGCChan: + case <-time.After(10 * time.Second): + t.Fatal("updateGC was not called after getting chunk with ModeGetRequest") + } + + // no need to wait for update gc hook anymore + resetTestHookUpdateGC() + // upload and sync another chunk to trigger // garbage collection - chunk := generateRandomChunk() - err = uploader.Put(chunk) + ch := generateTestRandomChunk() + err = uploader.Put(ch) if err != nil { t.Fatal(err) } - err = syncer.Set(chunk.Address()) + err = syncer.Set(ch.Address()) if err != nil { t.Fatal(err) } - addrs = append(addrs, chunk.Address()) + addrs = append(addrs, ch.Address()) // wait for garbage collection @@ -217,8 +236,8 @@ func TestDB_collectGarbageWorker_withRequests(t *testing.T) { // the second synced chunk should be removed t.Run("get gc-ed chunk", func(t *testing.T) { _, err := db.NewGetter(ModeGetRequest).Get(addrs[1]) - if err != storage.ErrChunkNotFound { - t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound) + if err != chunk.ErrChunkNotFound { + t.Errorf("got error %v, want %v", err, chunk.ErrChunkNotFound) } }) @@ -254,7 +273,7 @@ func TestDB_gcSize(t *testing.T) { count := 100 for i := 0; i < count; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { diff --git a/swarm/storage/localstore/index_test.go b/swarm/storage/localstore/index_test.go index d9abf440f1..cf19e4f6cd 100644 --- a/swarm/storage/localstore/index_test.go +++ b/swarm/storage/localstore/index_test.go @@ -21,7 +21,7 @@ import ( "math/rand" "testing" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // TestDB_pullIndex validates the ordering of keys in pull index. @@ -43,7 +43,7 @@ func TestDB_pullIndex(t *testing.T) { // upload random chunks for i := 0; i < chunkCount; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { @@ -62,8 +62,8 @@ func TestDB_pullIndex(t *testing.T) { } testItemsOrder(t, db.pullIndex, chunks, func(i, j int) (less bool) { - poi := storage.Proximity(db.baseKey, chunks[i].Address()) - poj := storage.Proximity(db.baseKey, chunks[j].Address()) + poi := chunk.Proximity(db.baseKey, chunks[i].Address()) + poj := chunk.Proximity(db.baseKey, chunks[j].Address()) if poi < poj { return true } @@ -95,7 +95,7 @@ func TestDB_gcIndex(t *testing.T) { // upload random chunks for i := 0; i < chunkCount; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index f92a9c1f2a..a66130fd30 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -24,8 +24,8 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/storage/mock" ) @@ -392,8 +392,8 @@ func (db *DB) Close() (err error) { // po computes the proximity order between the address // and database base key. -func (db *DB) po(addr storage.Address) (bin uint8) { - return uint8(storage.Proximity(db.baseKey, addr)) +func (db *DB) po(addr chunk.Address) (bin uint8) { + return uint8(chunk.Proximity(db.baseKey, addr)) } var ( @@ -409,7 +409,7 @@ var ( // If the address is locked this function will check it // in a for loop for addressLockTimeout time, after which // it will return ErrAddressLockTimeout error. -func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) { +func (db *DB) lockAddr(addr chunk.Address) (unlock func(), err error) { start := time.Now() lockKey := hex.EncodeToString(addr) for { @@ -426,7 +426,7 @@ func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) { } // chunkToItem creates new Item with data provided by the Chunk. -func chunkToItem(ch storage.Chunk) shed.Item { +func chunkToItem(ch chunk.Chunk) shed.Item { return shed.Item{ Address: ch.Address(), Data: ch.Data(), @@ -434,7 +434,7 @@ func chunkToItem(ch storage.Chunk) shed.Item { } // addressToItem creates new Item with a provided address. -func addressToItem(addr storage.Address) shed.Item { +func addressToItem(addr chunk.Address) shed.Item { return shed.Item{ Address: addr, } diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go index 6954b139a2..d106241733 100644 --- a/swarm/storage/localstore/localstore_test.go +++ b/swarm/storage/localstore/localstore_test.go @@ -29,9 +29,8 @@ import ( "testing" "time" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" "github.com/syndtr/goleveldb/leveldb" ) @@ -61,7 +60,7 @@ func TestDB(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { @@ -115,7 +114,7 @@ func TestDB_updateGCSem(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { @@ -188,7 +187,7 @@ func BenchmarkNew(b *testing.B) { uploader := db.NewPutter(ModePutUpload) syncer := db.NewSetter(ModeSetSync) for i := 0; i < count; i++ { - chunk := generateFakeRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { b.Fatal(err) @@ -251,53 +250,47 @@ func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) { return db, cleanupFunc } -// generateRandomChunk generates a valid Chunk with -// data size of default chunk size. -func generateRandomChunk() storage.Chunk { - return storage.GenerateRandomChunk(ch.DefaultSize) -} - func init() { - // needed for generateFakeRandomChunk + // needed for generateTestRandomChunk rand.Seed(time.Now().UnixNano()) } -// generateFakeRandomChunk generates a Chunk that is not +// generateTestRandomChunk generates a Chunk that is not // valid, but it contains a random key and a random value. -// This function is faster then storage.GenerateRandomChunk +// This function is faster then storage.generateTestRandomChunk // which generates a valid chunk. // Some tests in this package do not need valid chunks, just // random data, and their execution time can be decreased // using this function. -func generateFakeRandomChunk() storage.Chunk { - data := make([]byte, ch.DefaultSize) +func generateTestRandomChunk() chunk.Chunk { + data := make([]byte, chunk.DefaultSize) rand.Read(data) key := make([]byte, 32) rand.Read(key) - return storage.NewChunk(key, data) + return chunk.NewChunk(key, data) } -// TestGenerateFakeRandomChunk validates that -// generateFakeRandomChunk returns random data by comparing +// TestGenerateTestRandomChunk validates that +// generateTestRandomChunk returns random data by comparing // two generated chunks. -func TestGenerateFakeRandomChunk(t *testing.T) { - c1 := generateFakeRandomChunk() - c2 := generateFakeRandomChunk() +func TestGenerateTestRandomChunk(t *testing.T) { + c1 := generateTestRandomChunk() + c2 := generateTestRandomChunk() addrLen := len(c1.Address()) if addrLen != 32 { t.Errorf("first chunk address length %v, want %v", addrLen, 32) } dataLen := len(c1.Data()) - if dataLen != ch.DefaultSize { - t.Errorf("first chunk data length %v, want %v", dataLen, ch.DefaultSize) + if dataLen != chunk.DefaultSize { + t.Errorf("first chunk data length %v, want %v", dataLen, chunk.DefaultSize) } addrLen = len(c2.Address()) if addrLen != 32 { t.Errorf("second chunk address length %v, want %v", addrLen, 32) } dataLen = len(c2.Data()) - if dataLen != ch.DefaultSize { - t.Errorf("second chunk data length %v, want %v", dataLen, ch.DefaultSize) + if dataLen != chunk.DefaultSize { + t.Errorf("second chunk data length %v, want %v", dataLen, chunk.DefaultSize) } if bytes.Equal(c1.Address(), c2.Address()) { t.Error("fake chunks addresses do not differ") @@ -309,7 +302,7 @@ func TestGenerateFakeRandomChunk(t *testing.T) { // newRetrieveIndexesTest returns a test function that validates if the right // chunk values are in the retrieval indexes. -func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { +func newRetrieveIndexesTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { return func(t *testing.T) { item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) if err != nil { @@ -328,7 +321,7 @@ func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessT // newRetrieveIndexesTestWithAccess returns a test function that validates if the right // chunk values are in the retrieval indexes when access time must be stored. -func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { +func newRetrieveIndexesTestWithAccess(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { return func(t *testing.T) { item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) if err != nil { @@ -348,7 +341,7 @@ func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestam // newPullIndexTest returns a test function that validates if the right // chunk values are in the pull index. -func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { +func newPullIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { return func(t *testing.T) { item, err := db.pullIndex.Get(shed.Item{ Address: chunk.Address(), @@ -365,7 +358,7 @@ func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr // newPushIndexTest returns a test function that validates if the right // chunk values are in the push index. -func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { +func newPushIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { return func(t *testing.T) { item, err := db.pushIndex.Get(shed.Item{ Address: chunk.Address(), @@ -382,7 +375,7 @@ func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantErr // newGCIndexTest returns a test function that validates if the right // chunk values are in the push index. -func newGCIndexTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { +func newGCIndexTest(db *DB, chunk chunk.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { return func(t *testing.T) { item, err := db.gcIndex.Get(shed.Item{ Address: chunk.Address(), @@ -436,7 +429,7 @@ func newIndexGCSizeTest(db *DB) func(t *testing.T) { // testIndexChunk embeds storageChunk with additional data that is stored // in database. It is used for index values validations. type testIndexChunk struct { - storage.Chunk + chunk.Chunk storeTimestamp int64 } diff --git a/swarm/storage/localstore/mode_get.go b/swarm/storage/localstore/mode_get.go index 3a69f6e9d4..9640cd27e1 100644 --- a/swarm/storage/localstore/mode_get.go +++ b/swarm/storage/localstore/mode_get.go @@ -18,8 +18,8 @@ package localstore import ( "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" "github.com/syndtr/goleveldb/leveldb" ) @@ -51,23 +51,23 @@ func (db *DB) NewGetter(mode ModeGet) *Getter { } // Get returns a chunk from the database. If the chunk is -// not found storage.ErrChunkNotFound will be returned. +// not found chunk.ErrChunkNotFound will be returned. // All required indexes will be updated required by the // Getter Mode. -func (g *Getter) Get(addr storage.Address) (chunk storage.Chunk, err error) { +func (g *Getter) Get(addr chunk.Address) (ch chunk.Chunk, err error) { out, err := g.db.get(g.mode, addr) if err != nil { if err == leveldb.ErrNotFound { - return nil, storage.ErrChunkNotFound + return nil, chunk.ErrChunkNotFound } return nil, err } - return storage.NewChunk(out.Address, out.Data), nil + return chunk.NewChunk(out.Address, out.Data), nil } // get returns Item from the retrieval index // and updates other indexes. -func (db *DB) get(mode ModeGet, addr storage.Address) (out shed.Item, err error) { +func (db *DB) get(mode ModeGet, addr chunk.Address) (out shed.Item, err error) { item := addressToItem(addr) out, err = db.retrievalDataIndex.Get(item) diff --git a/swarm/storage/localstore/mode_get_test.go b/swarm/storage/localstore/mode_get_test.go index 6615a3b889..28a70ee0cc 100644 --- a/swarm/storage/localstore/mode_get_test.go +++ b/swarm/storage/localstore/mode_get_test.go @@ -32,7 +32,7 @@ func TestModeGetRequest(t *testing.T) { return uploadTimestamp })() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { @@ -146,7 +146,7 @@ func TestModeGetSync(t *testing.T) { return uploadTimestamp })() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go index 1a5a3d1b10..81df435351 100644 --- a/swarm/storage/localstore/mode_put.go +++ b/swarm/storage/localstore/mode_put.go @@ -17,8 +17,8 @@ package localstore import ( + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" "github.com/syndtr/goleveldb/leveldb" ) @@ -53,7 +53,7 @@ func (db *DB) NewPutter(mode ModePut) *Putter { // Put stores the Chunk to database and depending // on the Putter mode, it updates required indexes. -func (p *Putter) Put(ch storage.Chunk) (err error) { +func (p *Putter) Put(ch chunk.Chunk) (err error) { return p.db.put(p.mode, chunkToItem(ch)) } diff --git a/swarm/storage/localstore/mode_put_test.go b/swarm/storage/localstore/mode_put_test.go index ffe6a4cb4e..8ecae1d2e9 100644 --- a/swarm/storage/localstore/mode_put_test.go +++ b/swarm/storage/localstore/mode_put_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // TestModePutRequest validates ModePutRequest index values on the provided DB. @@ -33,7 +33,7 @@ func TestModePutRequest(t *testing.T) { putter := db.NewPutter(ModePutRequest) - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() // keep the record when the chunk is stored var storeTimestamp int64 @@ -87,7 +87,7 @@ func TestModePutSync(t *testing.T) { return wantTimestamp })() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutSync).Put(chunk) if err != nil { @@ -109,7 +109,7 @@ func TestModePutUpload(t *testing.T) { return wantTimestamp })() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { @@ -132,7 +132,7 @@ func TestModePutUpload_parallel(t *testing.T) { chunkCount := 1000 workerCount := 100 - chunkChan := make(chan storage.Chunk) + chunkChan := make(chan chunk.Chunk) errChan := make(chan error) doneChan := make(chan struct{}) defer close(doneChan) @@ -159,13 +159,13 @@ func TestModePutUpload_parallel(t *testing.T) { }(i) } - chunks := make([]storage.Chunk, 0) + chunks := make([]chunk.Chunk, 0) var chunksMu sync.Mutex // send chunks to workers go func() { for i := 0; i < chunkCount; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() select { case chunkChan <- chunk: case <-doneChan: @@ -271,9 +271,9 @@ func benchmarkPutUpload(b *testing.B, o *Options, count, maxParallelUploads int) defer cleanupFunc() uploader := db.NewPutter(ModePutUpload) - chunks := make([]storage.Chunk, count) + chunks := make([]chunk.Chunk, count) for i := 0; i < count; i++ { - chunks[i] = generateFakeRandomChunk() + chunks[i] = generateTestRandomChunk() } errs := make(chan error) b.StartTimer() diff --git a/swarm/storage/localstore/mode_set.go b/swarm/storage/localstore/mode_set.go index a522f4447c..a7c9875fed 100644 --- a/swarm/storage/localstore/mode_set.go +++ b/swarm/storage/localstore/mode_set.go @@ -17,7 +17,7 @@ package localstore import ( - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/syndtr/goleveldb/leveldb" ) @@ -53,7 +53,7 @@ func (db *DB) NewSetter(mode ModeSet) *Setter { // Set updates database indexes for a specific // chunk represented by the address. -func (s *Setter) Set(addr storage.Address) (err error) { +func (s *Setter) Set(addr chunk.Address) (err error) { return s.db.set(s.mode, addr) } @@ -61,7 +61,7 @@ func (s *Setter) Set(addr storage.Address) (err error) { // chunk represented by the address. // It acquires lockAddr to protect two calls // of this function for the same address in parallel. -func (db *DB) set(mode ModeSet, addr storage.Address) (err error) { +func (db *DB) set(mode ModeSet, addr chunk.Address) (err error) { // protect parallel updates unlock, err := db.lockAddr(addr) if err != nil { diff --git a/swarm/storage/localstore/mode_set_test.go b/swarm/storage/localstore/mode_set_test.go index 94cd0a3e2c..674aaabecd 100644 --- a/swarm/storage/localstore/mode_set_test.go +++ b/swarm/storage/localstore/mode_set_test.go @@ -28,7 +28,7 @@ func TestModeSetAccess(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() wantTimestamp := time.Now().UTC().UnixNano() defer setNow(func() (t int64) { @@ -56,7 +56,7 @@ func TestModeSetSync(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() wantTimestamp := time.Now().UTC().UnixNano() defer setNow(func() (t int64) { @@ -89,7 +89,7 @@ func TestModeSetRemove(t *testing.T) { db, cleanupFunc := newTestDB(t, nil) defer cleanupFunc() - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := db.NewPutter(ModePutUpload).Put(chunk) if err != nil { diff --git a/swarm/storage/localstore/retrieval_index_test.go b/swarm/storage/localstore/retrieval_index_test.go index 9f5b452c5c..b08790124b 100644 --- a/swarm/storage/localstore/retrieval_index_test.go +++ b/swarm/storage/localstore/retrieval_index_test.go @@ -20,7 +20,7 @@ import ( "strconv" "testing" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // BenchmarkRetrievalIndexes uploads a number of chunks in order to measure @@ -64,9 +64,9 @@ func benchmarkRetrievalIndexes(b *testing.B, o *Options, count int) { uploader := db.NewPutter(ModePutUpload) syncer := db.NewSetter(ModeSetSync) requester := db.NewGetter(ModeGetRequest) - addrs := make([]storage.Address, count) + addrs := make([]chunk.Address, count) for i := 0; i < count; i++ { - chunk := generateFakeRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { b.Fatal(err) @@ -134,9 +134,9 @@ func benchmarkUpload(b *testing.B, o *Options, count int) { db, cleanupFunc := newTestDB(b, o) defer cleanupFunc() uploader := db.NewPutter(ModePutUpload) - chunks := make([]storage.Chunk, count) + chunks := make([]chunk.Chunk, count) for i := 0; i < count; i++ { - chunk := generateFakeRandomChunk() + chunk := generateTestRandomChunk() chunks[i] = chunk } b.StartTimer() diff --git a/swarm/storage/localstore/subscription_pull.go b/swarm/storage/localstore/subscription_pull.go index a18f0915d9..0830eee70f 100644 --- a/swarm/storage/localstore/subscription_pull.go +++ b/swarm/storage/localstore/subscription_pull.go @@ -24,8 +24,8 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" ) // SubscribePull returns a channel that provides chunk addresses and stored times from pull syncing index. @@ -161,7 +161,7 @@ func (db *DB) SubscribePull(ctx context.Context, bin uint8, since, until *ChunkD // ChunkDescriptor holds information required for Pull syncing. This struct // is provided by subscribing to pull index. type ChunkDescriptor struct { - Address storage.Address + Address chunk.Address StoreTimestamp int64 } diff --git a/swarm/storage/localstore/subscription_pull_test.go b/swarm/storage/localstore/subscription_pull_test.go index 9800329eaa..130f0c9fe7 100644 --- a/swarm/storage/localstore/subscription_pull_test.go +++ b/swarm/storage/localstore/subscription_pull_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // TestDB_SubscribePull uploads some chunks before and after @@ -37,7 +37,7 @@ func TestDB_SubscribePull(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make(map[uint8][]storage.Address) + addrs := make(map[uint8][]chunk.Address) var addrsMu sync.Mutex var wantedChunksCount int @@ -53,7 +53,7 @@ func TestDB_SubscribePull(t *testing.T) { // to validate the number of addresses received by the subscription errChan := make(chan error) - for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ { ch, stop := db.SubscribePull(ctx, bin, nil, nil) defer stop() @@ -84,7 +84,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make(map[uint8][]storage.Address) + addrs := make(map[uint8][]chunk.Address) var addrsMu sync.Mutex var wantedChunksCount int @@ -105,7 +105,7 @@ func TestDB_SubscribePull_multiple(t *testing.T) { // start a number of subscriptions // that all of them will write every address error to errChan for j := 0; j < subsCount; j++ { - for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ { ch, stop := db.SubscribePull(ctx, bin, nil, nil) defer stop() @@ -137,7 +137,7 @@ func TestDB_SubscribePull_since(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make(map[uint8][]storage.Address) + addrs := make(map[uint8][]chunk.Address) var addrsMu sync.Mutex var wantedChunksCount int @@ -156,20 +156,20 @@ func TestDB_SubscribePull_since(t *testing.T) { last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { - chunk := generateRandomChunk() + ch := generateTestRandomChunk() - err := uploader.Put(chunk) + err := uploader.Put(ch) if err != nil { t.Fatal(err) } - bin := db.po(chunk.Address()) + bin := db.po(ch.Address()) if _, ok := addrs[bin]; !ok { - addrs[bin] = make([]storage.Address, 0) + addrs[bin] = make([]chunk.Address, 0) } if wanted { - addrs[bin] = append(addrs[bin], chunk.Address()) + addrs[bin] = append(addrs[bin], ch.Address()) wantedChunksCount++ } @@ -178,7 +178,7 @@ func TestDB_SubscribePull_since(t *testing.T) { lastTimestampMu.RUnlock() last[bin] = ChunkDescriptor{ - Address: chunk.Address(), + Address: ch.Address(), StoreTimestamp: storeTimestamp, } } @@ -199,7 +199,7 @@ func TestDB_SubscribePull_since(t *testing.T) { // to validate the number of addresses received by the subscription errChan := make(chan error) - for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ { var since *ChunkDescriptor if c, ok := last[bin]; ok { since = &c @@ -228,7 +228,7 @@ func TestDB_SubscribePull_until(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make(map[uint8][]storage.Address) + addrs := make(map[uint8][]chunk.Address) var addrsMu sync.Mutex var wantedChunksCount int @@ -247,20 +247,20 @@ func TestDB_SubscribePull_until(t *testing.T) { last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { - chunk := generateRandomChunk() + ch := generateTestRandomChunk() - err := uploader.Put(chunk) + err := uploader.Put(ch) if err != nil { t.Fatal(err) } - bin := db.po(chunk.Address()) + bin := db.po(ch.Address()) if _, ok := addrs[bin]; !ok { - addrs[bin] = make([]storage.Address, 0) + addrs[bin] = make([]chunk.Address, 0) } if wanted { - addrs[bin] = append(addrs[bin], chunk.Address()) + addrs[bin] = append(addrs[bin], ch.Address()) wantedChunksCount++ } @@ -269,7 +269,7 @@ func TestDB_SubscribePull_until(t *testing.T) { lastTimestampMu.RUnlock() last[bin] = ChunkDescriptor{ - Address: chunk.Address(), + Address: ch.Address(), StoreTimestamp: storeTimestamp, } } @@ -290,7 +290,7 @@ func TestDB_SubscribePull_until(t *testing.T) { // to validate the number of addresses received by the subscription errChan := make(chan error) - for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ { until, ok := last[bin] if !ok { continue @@ -318,7 +318,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make(map[uint8][]storage.Address) + addrs := make(map[uint8][]chunk.Address) var addrsMu sync.Mutex var wantedChunksCount int @@ -337,20 +337,20 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { last = make(map[uint8]ChunkDescriptor) for i := 0; i < count; i++ { - chunk := generateRandomChunk() + ch := generateTestRandomChunk() - err := uploader.Put(chunk) + err := uploader.Put(ch) if err != nil { t.Fatal(err) } - bin := db.po(chunk.Address()) + bin := db.po(ch.Address()) if _, ok := addrs[bin]; !ok { - addrs[bin] = make([]storage.Address, 0) + addrs[bin] = make([]chunk.Address, 0) } if wanted { - addrs[bin] = append(addrs[bin], chunk.Address()) + addrs[bin] = append(addrs[bin], ch.Address()) wantedChunksCount++ } @@ -359,7 +359,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { lastTimestampMu.RUnlock() last[bin] = ChunkDescriptor{ - Address: chunk.Address(), + Address: ch.Address(), StoreTimestamp: storeTimestamp, } } @@ -386,7 +386,7 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { // to validate the number of addresses received by the subscription errChan := make(chan error) - for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + for bin := uint8(0); bin <= uint8(chunk.MaxPO); bin++ { var since *ChunkDescriptor if c, ok := upload1[bin]; ok { since = &c @@ -412,23 +412,23 @@ func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { // uploadRandomChunksBin uploads random chunks to database and adds them to // the map of addresses ber bin. -func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) { +func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) { addrsMu.Lock() defer addrsMu.Unlock() for i := 0; i < count; i++ { - chunk := generateRandomChunk() + ch := generateTestRandomChunk() - err := uploader.Put(chunk) + err := uploader.Put(ch) if err != nil { t.Fatal(err) } - bin := db.po(chunk.Address()) + bin := db.po(ch.Address()) if _, ok := addrs[bin]; !ok { - addrs[bin] = make([]storage.Address, 0) + addrs[bin] = make([]chunk.Address, 0) } - addrs[bin] = append(addrs[bin], chunk.Address()) + addrs[bin] = append(addrs[bin], ch.Address()) *wantedChunksCount++ } @@ -437,7 +437,7 @@ func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uin // readPullSubscriptionBin is a helper function that reads all ChunkDescriptors from a channel and // sends error to errChan, even if it is nil, to count the number of ChunkDescriptors // returned by the channel. -func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, errChan chan error) { +func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]chunk.Address, addrsMu *sync.Mutex, errChan chan error) { var i int // address index for { select { diff --git a/swarm/storage/localstore/subscription_push.go b/swarm/storage/localstore/subscription_push.go index b13f293998..5cbc2eb6ff 100644 --- a/swarm/storage/localstore/subscription_push.go +++ b/swarm/storage/localstore/subscription_push.go @@ -21,16 +21,16 @@ import ( "sync" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/shed" - "github.com/ethereum/go-ethereum/swarm/storage" ) // SubscribePush returns a channel that provides storage chunks with ordering from push syncing index. // Returned stop function will terminate current and further iterations, and also it will close // the returned channel without any errors. Make sure that you check the second returned parameter // from the channel to stop iteration when its value is false. -func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop func()) { - chunks := make(chan storage.Chunk) +func (db *DB) SubscribePush(ctx context.Context) (c <-chan chunk.Chunk, stop func()) { + chunks := make(chan chunk.Chunk) trigger := make(chan struct{}, 1) db.pushTriggersMu.Lock() @@ -65,7 +65,7 @@ func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop f } select { - case chunks <- storage.NewChunk(dataItem.Address, dataItem.Data): + case chunks <- chunk.NewChunk(dataItem.Address, dataItem.Data): // set next iteration start item // when its chunk is successfully sent to channel sinceItem = &item diff --git a/swarm/storage/localstore/subscription_push_test.go b/swarm/storage/localstore/subscription_push_test.go index 0c8d7d0b9a..30fb98eb21 100644 --- a/swarm/storage/localstore/subscription_push_test.go +++ b/swarm/storage/localstore/subscription_push_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/chunk" ) // TestDB_SubscribePush uploads some chunks before and after @@ -36,7 +36,7 @@ func TestDB_SubscribePush(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - chunks := make([]storage.Chunk, 0) + chunks := make([]chunk.Chunk, 0) var chunksMu sync.Mutex uploadRandomChunks := func(count int) { @@ -44,7 +44,7 @@ func TestDB_SubscribePush(t *testing.T) { defer chunksMu.Unlock() for i := 0; i < count; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { @@ -124,7 +124,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) { uploader := db.NewPutter(ModePutUpload) - addrs := make([]storage.Address, 0) + addrs := make([]chunk.Address, 0) var addrsMu sync.Mutex uploadRandomChunks := func(count int) { @@ -132,7 +132,7 @@ func TestDB_SubscribePush_multiple(t *testing.T) { defer addrsMu.Unlock() for i := 0; i < count; i++ { - chunk := generateRandomChunk() + chunk := generateTestRandomChunk() err := uploader.Put(chunk) if err != nil { diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index ec69951c4f..fcadcefa0b 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -23,7 +23,7 @@ import ( "testing" "time" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" ) var ( @@ -65,7 +65,7 @@ func TestValidator(t *testing.T) { // add content address validator and check puts // bad should fail, good should pass store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc)) - chunks = GenerateRandomChunks(ch.DefaultSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.Data(), goodChunk.Data()) @@ -83,7 +83,7 @@ func TestValidator(t *testing.T) { var negV boolTestValidator store.Validators = append(store.Validators, negV) - chunks = GenerateRandomChunks(ch.DefaultSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.Data(), goodChunk.Data()) @@ -101,7 +101,7 @@ func TestValidator(t *testing.T) { var posV boolTestValidator = true store.Validators = append(store.Validators, posV) - chunks = GenerateRandomChunks(ch.DefaultSize, 2) + chunks = GenerateRandomChunks(chunk.DefaultSize, 2) goodChunk = chunks[0] badChunk = chunks[1] copy(badChunk.Data(), goodChunk.Data()) @@ -138,7 +138,7 @@ func putChunks(store *LocalStore, chunks ...Chunk) []error { func put(store *LocalStore, n int, f func(i int64) Chunk) (hs []Address, errs []error) { for i := int64(0); i < int64(n); i++ { - chunk := f(ch.DefaultSize) + chunk := f(chunk.DefaultSize) err := store.Put(context.TODO(), chunk) errs = append(errs, err) hs = append(hs, chunk.Address()) @@ -158,7 +158,7 @@ func TestGetFrequentlyAccessedChunkWontGetGarbageCollected(t *testing.T) { var chunks []Chunk for i := 0; i < ldbCap; i++ { - chunks = append(chunks, GenerateRandomChunk(ch.DefaultSize)) + chunks = append(chunks, GenerateRandomChunk(chunk.DefaultSize)) } mostAccessed := chunks[0].Address() diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index 88ec6c28f4..6538776259 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/enode" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" ) var sourcePeerID = enode.HexID("99d8594b52298567d2ca3f4c441a5ba0140ee9245e26460d01102a52773c73b9") @@ -114,7 +114,7 @@ func mustNewNetStoreWithFetcher(t *testing.T) (*NetStore, *mockNetFetcher) { func TestNetStoreGetAndPut(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -174,7 +174,7 @@ func TestNetStoreGetAndPut(t *testing.T) { func TestNetStoreGetAfterPut(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() @@ -209,7 +209,7 @@ func TestNetStoreGetAfterPut(t *testing.T) { func TestNetStoreGetTimeout(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() @@ -261,7 +261,7 @@ func TestNetStoreGetTimeout(t *testing.T) { func TestNetStoreGetCancel(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) @@ -313,7 +313,7 @@ func TestNetStoreGetCancel(t *testing.T) { func TestNetStoreMultipleGetAndPut(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -387,7 +387,7 @@ func TestNetStoreMultipleGetAndPut(t *testing.T) { func TestNetStoreFetchFuncTimeout(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) defer cancel() @@ -426,7 +426,7 @@ func TestNetStoreFetchFuncTimeout(t *testing.T) { func TestNetStoreFetchFuncAfterPut(t *testing.T) { netStore := mustNewNetStore(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -453,7 +453,7 @@ func TestNetStoreFetchFuncAfterPut(t *testing.T) { func TestNetStoreGetCallsRequest(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx := context.WithValue(context.Background(), "hopcount", uint8(5)) ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) @@ -481,7 +481,7 @@ func TestNetStoreGetCallsRequest(t *testing.T) { func TestNetStoreGetCallsOffer(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) // If a source peer is added to the context, NetStore will handle it as an offer ctx := context.WithValue(context.Background(), "source", sourcePeerID.String()) @@ -567,7 +567,7 @@ func TestNetStoreFetcherCountPeers(t *testing.T) { func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) defer cancel() @@ -632,7 +632,7 @@ func TestNetStoreFetchFuncCalledMultipleTimes(t *testing.T) { func TestNetStoreFetcherLifeCycleWithTimeout(t *testing.T) { netStore, fetcher := mustNewNetStoreWithFetcher(t) - chunk := GenerateRandomChunk(ch.DefaultSize) + chunk := GenerateRandomChunk(chunk.DefaultSize) ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/swarm/storage/pyramid.go b/swarm/storage/pyramid.go index ed0f843b93..281bbe9fe3 100644 --- a/swarm/storage/pyramid.go +++ b/swarm/storage/pyramid.go @@ -25,7 +25,7 @@ import ( "sync" "time" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "github.com/ethereum/go-ethereum/swarm/log" ) @@ -97,11 +97,11 @@ func NewPyramidSplitterParams(addr Address, reader io.Reader, putter Putter, get New chunks to store are store using the putter which the caller provides. */ func PyramidSplit(ctx context.Context, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, ch.DefaultSize)).Split(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(nil, reader, putter, getter, chunk.DefaultSize)).Split(ctx) } func PyramidAppend(ctx context.Context, addr Address, reader io.Reader, putter Putter, getter Getter) (Address, func(context.Context) error, error) { - return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, ch.DefaultSize)).Append(ctx) + return NewPyramidSplitter(NewPyramidSplitterParams(addr, reader, putter, getter, chunk.DefaultSize)).Append(ctx) } // Entry to create a tree node diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 7ec21328e2..2f39685b42 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -22,53 +22,29 @@ import ( "crypto" "crypto/rand" "encoding/binary" - "fmt" "io" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/swarm/bmt" - ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/chunk" "golang.org/x/crypto/sha3" ) -const MaxPO = 16 -const AddressLength = 32 +// MaxPO is the same as chunk.MaxPO for backward compatibility. +const MaxPO = chunk.MaxPO + +// AddressLength is the same as chunk.AddressLength for backward compatibility. +const AddressLength = chunk.AddressLength type SwarmHasher func() SwarmHash -type Address []byte +// Address is an alias for chunk.Address for backward compatibility. +type Address = chunk.Address -// Proximity(x, y) returns the proximity order of the MSB distance between x and y -// -// The distance metric MSB(x, y) of two equal length byte sequences x an y is the -// value of the binary integer cast of the x^y, ie., x and y bitwise xor-ed. -// the binary cast is big endian: most significant bit first (=MSB). -// -// Proximity(x, y) is a discrete logarithmic scaling of the MSB distance. -// It is defined as the reverse rank of the integer part of the base 2 -// logarithm of the distance. -// It is calculated by counting the number of common leading zeros in the (MSB) -// binary representation of the x^y. -// -// (0 farthest, 255 closest, 256 self) -func Proximity(one, other []byte) (ret int) { - b := (MaxPO-1)/8 + 1 - if b > len(one) { - b = len(one) - } - m := 8 - for i := 0; i < b; i++ { - oxo := one[i] ^ other[i] - for j := 0; j < m; j++ { - if (oxo>>uint8(7-j))&0x01 != 0 { - return i*8 + j - } - } - } - return MaxPO -} +// Proximity is the same as chunk.Proximity for backward compatibility. +var Proximity = chunk.Proximity -var ZeroAddr = Address(common.Hash{}.Bytes()) +// ZeroAddr is the same as chunk.ZeroAddr for backward compatibility. +var ZeroAddr = chunk.ZeroAddr func MakeHashFunc(hash string) SwarmHasher { switch hash { @@ -80,7 +56,7 @@ func MakeHashFunc(hash string) SwarmHasher { return func() SwarmHash { hasher := sha3.NewLegacyKeccak256 hasherSize := hasher().Size() - segmentCount := ch.DefaultSize / hasherSize + segmentCount := chunk.DefaultSize / hasherSize pool := bmt.NewTreePool(hasher, segmentCount, bmt.PoolSize) return bmt.New(pool) } @@ -88,33 +64,6 @@ func MakeHashFunc(hash string) SwarmHasher { return nil } -func (a Address) Hex() string { - return fmt.Sprintf("%064x", []byte(a[:])) -} - -func (a Address) Log() string { - if len(a[:]) < 8 { - return fmt.Sprintf("%x", []byte(a[:])) - } - return fmt.Sprintf("%016x", []byte(a[:8])) -} - -func (a Address) String() string { - return fmt.Sprintf("%064x", []byte(a)) -} - -func (a Address) MarshalJSON() (out []byte, err error) { - return []byte(`"` + a.String() + `"`), nil -} - -func (a *Address) UnmarshalJSON(value []byte) error { - s := string(value) - *a = make([]byte, 32) - h := common.Hex2Bytes(s[1 : len(s)-1]) - copy(*a, h) - return nil -} - type AddressCollection []Address func NewAddressCollection(l int) AddressCollection { @@ -133,38 +82,11 @@ func (c AddressCollection) Swap(i, j int) { c[i], c[j] = c[j], c[i] } -// Chunk interface implemented by context.Contexts and data chunks -type Chunk interface { - Address() Address - Data() []byte -} +// Chunk is an alias for chunk.Chunk for backward compatibility. +type Chunk = chunk.Chunk -type chunk struct { - addr Address - sdata []byte - span int64 -} - -func NewChunk(addr Address, data []byte) *chunk { - return &chunk{ - addr: addr, - sdata: data, - span: -1, - } -} - -func (c *chunk) Address() Address { - return c.addr -} - -func (c *chunk) Data() []byte { - return c.sdata -} - -// String() for pretty printing -func (self *chunk) String() string { - return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata)) -} +// NewChunk is the same as chunk.NewChunk for backward compatibility. +var NewChunk = chunk.NewChunk func GenerateRandomChunk(dataSize int64) Chunk { hasher := MakeHashFunc(DefaultHash)() @@ -274,9 +196,9 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { } // Validate that the given key is a valid content address for the given data -func (v *ContentAddressValidator) Validate(chunk Chunk) bool { - data := chunk.Data() - if l := len(data); l < 9 || l > ch.DefaultSize+8 { +func (v *ContentAddressValidator) Validate(ch Chunk) bool { + data := ch.Data() + if l := len(data); l < 9 || l > chunk.DefaultSize+8 { // log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l) return false } @@ -286,7 +208,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool { hasher.Write(data[8:]) hash := hasher.Sum(nil) - return bytes.Equal(hash, chunk.Address()) + return bytes.Equal(hash, ch.Address()) } type ChunkStore interface { From 7ebd2fa5dbbc2944eae5bf5844d26c9840e7b12e Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 27 Feb 2019 20:10:10 +0800 Subject: [PATCH 19/21] vendor: update leveldb upstream which include a compaction fix (#19163) --- .../github.com/syndtr/goleveldb/leveldb/db.go | 28 ++++++++++--- .../syndtr/goleveldb/leveldb/db_compaction.go | 2 +- .../syndtr/goleveldb/leveldb/db_snapshot.go | 4 ++ .../goleveldb/leveldb/db_transaction.go | 6 ++- .../syndtr/goleveldb/leveldb/session.go | 6 +-- .../syndtr/goleveldb/leveldb/table.go | 8 ++++ .../syndtr/goleveldb/leveldb/version.go | 41 +++++++++++++++++-- vendor/vendor.json | 6 +-- 8 files changed, 84 insertions(+), 17 deletions(-) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index b27c38d37e..0de5ffe8d7 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -468,7 +468,7 @@ func recoverTable(s *session, o *opt.Options) error { } // Commit. - return s.commit(rec) + return s.commit(rec, false) } func (db *DB) recoverJournal() error { @@ -538,7 +538,7 @@ func (db *DB) recoverJournal() error { rec.setJournalNum(fd.Num) rec.setSeqNum(db.seq) - if err := db.s.commit(rec); err != nil { + if err := db.s.commit(rec, false); err != nil { fr.Close() return err } @@ -617,7 +617,7 @@ func (db *DB) recoverJournal() error { // Commit. rec.setJournalNum(db.journalFd.Num) rec.setSeqNum(db.seq) - if err := db.s.commit(rec); err != nil { + if err := db.s.commit(rec, false); err != nil { // Close journal on error. if db.journal != nil { db.journal.Close() @@ -872,6 +872,10 @@ func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { // DB. And a nil Range.Limit is treated as a key after all keys in // the DB. // +// WARNING: Any slice returned by interator (e.g. slice returned by calling +// Iterator.Key() or Iterator.Key() methods), its content should not be modified +// unless noted otherwise. +// // The iterator must be released after use, by calling Release method. // // Also read Iterator documentation of the leveldb/iterator package. @@ -953,15 +957,27 @@ func (db *DB) GetProperty(name string) (value string, err error) { value = "Compactions\n" + " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" + "-------+------------+---------------+---------------+---------------+---------------\n" + var totalTables int + var totalSize, totalRead, totalWrite int64 + var totalDuration time.Duration for level, tables := range v.levels { duration, read, write := db.compStats.getStat(level) if len(tables) == 0 && duration == 0 { continue } + totalTables += len(tables) + totalSize += tables.size() + totalRead += read + totalWrite += write + totalDuration += duration value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(), float64(read)/1048576.0, float64(write)/1048576.0) } + value += "-------+------------+---------------+---------------+---------------+---------------\n" + value += fmt.Sprintf(" Total | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", + totalTables, float64(totalSize)/1048576.0, totalDuration.Seconds(), + float64(totalRead)/1048576.0, float64(totalWrite)/1048576.0) case p == "iostats": value = fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f", float64(db.s.stor.reads())/1048576.0, @@ -1013,10 +1029,10 @@ type DBStats struct { BlockCacheSize int OpenedTablesCount int - LevelSizes []int64 + LevelSizes Sizes LevelTablesCounts []int - LevelRead []int64 - LevelWrite []int64 + LevelRead Sizes + LevelWrite Sizes LevelDurations []time.Duration } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 0c1b9a53b8..56f3632a7d 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -260,7 +260,7 @@ func (db *DB) compactionCommit(name string, rec *sessionRecord) { db.compCommitLk.Lock() defer db.compCommitLk.Unlock() // Defer is necessary. db.compactionTransactFunc(name+"@commit", func(cnt *compactionTransactCounter) error { - return db.s.commit(rec) + return db.s.commit(rec, true) }, nil) } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go index 2c69d2e531..c2ad70c847 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go @@ -142,6 +142,10 @@ func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) // DB. And a nil Range.Limit is treated as a key after all keys in // the DB. // +// WARNING: Any slice returned by interator (e.g. slice returned by calling +// Iterator.Key() or Iterator.Value() methods), its content should not be +// modified unless noted otherwise. +// // The iterator must be released after use, by calling Release method. // Releasing the snapshot doesn't mean releasing the iterator too, the // iterator would be still valid until released. diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go index b8f7e7d21d..f145b64fbb 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go @@ -69,6 +69,10 @@ func (tr *Transaction) Has(key []byte, ro *opt.ReadOptions) (bool, error) { // DB. And a nil Range.Limit is treated as a key after all keys in // the DB. // +// WARNING: Any slice returned by interator (e.g. slice returned by calling +// Iterator.Key() or Iterator.Key() methods), its content should not be modified +// unless noted otherwise. +// // The iterator must be released after use, by calling Release method. // // Also read Iterator documentation of the leveldb/iterator package. @@ -205,7 +209,7 @@ func (tr *Transaction) Commit() error { tr.stats.startTimer() var cerr error for retry := 0; retry < 3; retry++ { - cerr = tr.db.s.commit(&tr.rec) + cerr = tr.db.s.commit(&tr.rec, false) if cerr != nil { tr.db.logf("transaction@commit error R·%d %q", retry, cerr) select { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session.go b/vendor/github.com/syndtr/goleveldb/leveldb/session.go index 3f391f9346..1bec34c4c3 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/session.go @@ -180,19 +180,19 @@ func (s *session) recover() (err error) { } s.manifestFd = fd - s.setVersion(staging.finish()) + s.setVersion(staging.finish(false)) s.setNextFileNum(rec.nextFileNum) s.recordCommited(rec) return nil } // Commit session; need external synchronization. -func (s *session) commit(r *sessionRecord) (err error) { +func (s *session) commit(r *sessionRecord, trivial bool) (err error) { v := s.version() defer v.release() // spawn new version based on current version - nv := v.spawn(r) + nv := v.spawn(r, trivial) if s.manifest == nil { // manifest journal writer not yet created, create one diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table.go b/vendor/github.com/syndtr/goleveldb/leveldb/table.go index 1fac60d050..518e1db1cd 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/table.go @@ -150,6 +150,14 @@ func (tf tFiles) searchMax(icmp *iComparer, ikey internalKey) int { }) } +// Searches smallest index of tables whose its file number +// is smaller than the given number. +func (tf tFiles) searchNumLess(num int64) int { + return sort.Search(len(tf), func(i int) bool { + return tf[i].fd.Num < num + }) +} + // Returns true if given key range overlaps with one or more // tables key range. If unsorted is true then binary search will not be used. func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool { diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/version.go b/vendor/github.com/syndtr/goleveldb/leveldb/version.go index 73f272af5f..63b86fe545 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/version.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/version.go @@ -273,10 +273,10 @@ func (v *version) newStaging() *versionStaging { } // Spawn a new version based on this version. -func (v *version) spawn(r *sessionRecord) *version { +func (v *version) spawn(r *sessionRecord, trivial bool) *version { staging := v.newStaging() staging.commit(r) - return staging.finish() + return staging.finish(trivial) } func (v *version) fillRecord(r *sessionRecord) { @@ -446,7 +446,7 @@ func (p *versionStaging) commit(r *sessionRecord) { } } -func (p *versionStaging) finish() *version { +func (p *versionStaging) finish(trivial bool) *version { // Build new version. nv := newVersion(p.base.s) numLevel := len(p.levels) @@ -463,6 +463,12 @@ func (p *versionStaging) finish() *version { if level < len(p.levels) { scratch := p.levels[level] + // Short circuit if there is no change at all. + if len(scratch.added) == 0 && len(scratch.deleted) == 0 { + nv.levels[level] = baseTabels + continue + } + var nt tFiles // Prealloc list if possible. if n := len(baseTabels) + len(scratch.added) - len(scratch.deleted); n > 0 { @@ -480,6 +486,35 @@ func (p *versionStaging) finish() *version { nt = append(nt, t) } + // For normal table compaction, one compaction will only involve two levels + // of files. And the new files generated after merging the source level and + // source+1 level related files can be inserted as a whole into source+1 level + // without any overlap with the other source+1 files. + // + // When the amount of data maintained by leveldb is large, the number of files + // per level will be very large. While qsort is very inefficient for sorting + // already ordered arrays. Therefore, for the normal table compaction, we use + // binary search here to find the insert index to insert a batch of new added + // files directly instead of using qsort. + if trivial && len(scratch.added) > 0 { + added := make(tFiles, 0, len(scratch.added)) + for _, r := range scratch.added { + added = append(added, tableFileFromRecord(r)) + } + if level == 0 { + added.sortByNum() + index := nt.searchNumLess(added[len(added)-1].fd.Num) + nt = append(nt[:index], append(added, nt[index:]...)...) + } else { + added.sortByKey(p.base.s.icmp) + _, amax := added.getRange(p.base.s.icmp) + index := nt.searchMin(p.base.s.icmp, amax) + nt = append(nt[:index], append(added, nt[index:]...)...) + } + nv.levels[level] = nt + continue + } + // New tables. for _, r := range scratch.added { nt = append(nt, tableFileFromRecord(r)) diff --git a/vendor/vendor.json b/vendor/vendor.json index 0487e9299b..7827451668 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -455,10 +455,10 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "LV0VMVON7xY1ttV+s2ph83ntmDQ=", + "checksumSHA1": "4DuP8qJfeXFfdbcl4wr7l1VppcY=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", - "revisionTime": "2018-11-28T10:09:59Z" + "revision": "4217c9f31f5816db02addc94e56061da77f288d8", + "revisionTime": "2019-02-26T15:37:22Z" }, { "checksumSHA1": "mPNraL2edpk/2FYq26rSXfMHbJg=", From e43bc36226692fc128b5be38802efdd049e9db4a Mon Sep 17 00:00:00 2001 From: Samuel Marks Date: Tue, 26 Feb 2019 15:12:13 +1100 Subject: [PATCH 20/21] travis, appveyor, Dockerfile: upgrade to Go 1.12 --- .travis.yml | 31 +++++++++++++++++++--------- Dockerfile | 2 +- Dockerfile.alltools | 2 +- appveyor.yml | 4 ++-- core/tx_pool_test.go | 2 +- crypto/bn256/cloudflare/main_test.go | 2 +- crypto/bn256/google/main_test.go | 2 +- internal/build/util.go | 4 ++-- 8 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index b09bd7ce04..7d0777fee9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,6 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES - # These are the latest Go versions. - os: linux dist: trusty sudo: required @@ -26,8 +25,20 @@ matrix: - go run build/ci.go install - go run build/ci.go test -coverage $TEST_PACKAGES + # These are the latest Go versions. + - os: linux + dist: trusty + sudo: required + go: 1.12.x + script: + - sudo modprobe fuse + - sudo chmod 666 /dev/fuse + - sudo chown root:$USER /etc/fuse.conf + - go run build/ci.go install + - go run build/ci.go test -coverage $TEST_PACKAGES + - os: osx - go: 1.11.x + go: 1.12.x script: - echo "Increase the maximum number of open file descriptors on macOS" - NOFILE=20480 @@ -44,7 +55,7 @@ matrix: # This builder only tests code linters on latest version of Go - os: linux dist: trusty - go: 1.11.x + go: 1.12.x env: - lint git: @@ -56,7 +67,7 @@ matrix: - if: repo = ethereum/go-ethereum AND type = push os: linux dist: trusty - go: 1.11.x + go: 1.12.x env: - ubuntu-ppa git: @@ -79,7 +90,7 @@ matrix: os: linux dist: trusty sudo: required - go: 1.11.x + go: 1.12.x env: - azure-linux git: @@ -114,7 +125,7 @@ matrix: dist: trusty services: - docker - go: 1.11.x + go: 1.12.x env: - azure-linux-mips git: @@ -159,7 +170,7 @@ matrix: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://storage.googleapis.com/golang/go1.11.5.linux-amd64.tar.gz | tar -xz + - curl https://dl.google.com/go/go1.12.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go @@ -176,7 +187,7 @@ matrix: # This builder does the OSX Azure, iOS CocoaPods and iOS Azure uploads - if: repo = ethereum/go-ethereum AND type = push os: osx - go: 1.11.x + go: 1.12.x env: - azure-osx - azure-ios @@ -206,7 +217,7 @@ matrix: - if: repo = ethereum/go-ethereum AND type = cron os: linux dist: trusty - go: 1.11.x + go: 1.12.x env: - azure-purge git: @@ -218,7 +229,7 @@ matrix: if: repo = ethersphere/go-ethereum os: linux dist: trusty - go: 1.11.x + go: 1.12.x git: submodules: false # avoid cloning ethereum/tests script: ./build/travis_keepalive.sh go test -v -timeout 20m -race ./swarm... diff --git a/Dockerfile b/Dockerfile index e87dd35d32..4b205d6492 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build Geth in a stock Go builder container -FROM golang:1.11-alpine as builder +FROM golang:1.12-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers diff --git a/Dockerfile.alltools b/Dockerfile.alltools index e984a1b092..4a4a26f8bc 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -1,5 +1,5 @@ # Build Geth in a stock Go builder container -FROM golang:1.11-alpine as builder +FROM golang:1.12-alpine as builder RUN apk add --no-cache make gcc musl-dev linux-headers diff --git a/appveyor.yml b/appveyor.yml index 9ed4f114e3..515f3d9fbe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.11.5.windows-%GETH_ARCH%.zip - - 7z x go1.11.5.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://dl.google.com/go/go1.12.windows-%GETH_ARCH%.zip + - 7z x go1.12.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 7177ae9ce7..e1c2e06696 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -126,7 +126,7 @@ func validateEvents(events chan NewTxsEvent, count int) error { case ev := <-events: received = append(received, ev.Txs...) case <-time.After(time.Second): - return fmt.Errorf("event #%d not fired", received) + return fmt.Errorf("event #%d not fired", len(received)) } } if len(received) > count { diff --git a/crypto/bn256/cloudflare/main_test.go b/crypto/bn256/cloudflare/main_test.go index 0230f1b199..c0c85457be 100644 --- a/crypto/bn256/cloudflare/main_test.go +++ b/crypto/bn256/cloudflare/main_test.go @@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) { t.Error(err) continue } - t.Logf("%d: %x\n", n, g2.Marshal()) + t.Logf("%v: %x\n", n, g2.Marshal()) } } diff --git a/crypto/bn256/google/main_test.go b/crypto/bn256/google/main_test.go index 0230f1b199..c0c85457be 100644 --- a/crypto/bn256/google/main_test.go +++ b/crypto/bn256/google/main_test.go @@ -13,7 +13,7 @@ func TestRandomG2Marshal(t *testing.T) { t.Error(err) continue } - t.Logf("%d: %x\n", n, g2.Marshal()) + t.Logf("%v: %x\n", n, g2.Marshal()) } } diff --git a/internal/build/util.go b/internal/build/util.go index a41ecfbed3..b34371c2dd 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -143,9 +143,9 @@ func CopyFile(dst, src string, mode os.FileMode) { // so that go commands executed by build use the same version of Go as the 'host' that runs // build code. e.g. // -// /usr/lib/go-1.11/bin/go run build/ci.go ... +// /usr/lib/go-1.12/bin/go run build/ci.go ... // -// runs using go 1.11 and invokes go 1.11 tools from the same GOROOT. This is also important +// runs using go 1.12 and invokes go 1.12 tools from the same GOROOT. This is also important // because runtime.Version checks on the host should match the tools that are run. func GoTool(tool string, args ...string) *exec.Cmd { args = append([]string{tool}, args...) From 2b6158a51e4b4660ed4093880ed1e0a368aecce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 27 Feb 2019 14:20:29 +0200 Subject: [PATCH 21/21] common/fdlimit: fix macos file descriptors for Go 1.12 --- common/fdlimit/fdlimit_darwin.go | 71 +++++++++++++++++++++++++++++++ common/fdlimit/fdlimit_unix.go | 2 +- common/fdlimit/fdlimit_windows.go | 1 + 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 common/fdlimit/fdlimit_darwin.go diff --git a/common/fdlimit/fdlimit_darwin.go b/common/fdlimit/fdlimit_darwin.go new file mode 100644 index 0000000000..88dd0f56cb --- /dev/null +++ b/common/fdlimit/fdlimit_darwin.go @@ -0,0 +1,71 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fdlimit + +import "syscall" + +// hardlimit is the number of file descriptors allowed at max by the kernel. +const hardlimit = 10240 + +// Raise tries to maximize the file descriptor allowance of this process +// to the maximum hard-limit allowed by the OS. +// Returns the size it was set to (may differ from the desired 'max') +func Raise(max uint64) (uint64, error) { + // Get the current limit + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Try to update the limit to the max allowance + limit.Cur = limit.Max + if limit.Cur > max { + limit.Cur = max + } + if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // MacOS can silently apply further caps, so retrieve the actually set limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return limit.Cur, nil +} + +// Current retrieves the number of file descriptors allowed to be opened by this +// process. +func Current() (int, error) { + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return int(limit.Cur), nil +} + +// Maximum retrieves the maximum number of file descriptors this process is +// allowed to request for itself. +func Maximum() (int, error) { + // Retrieve the maximum allowed by dynamic OS limits + var limit syscall.Rlimit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + // Cap it to OPEN_MAX (10240) because macos is a special snowflake + if limit.Max > hardlimit { + limit.Max = hardlimit + } + return int(limit.Max), nil +} diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go index 6701127515..e5a575f7a7 100644 --- a/common/fdlimit/fdlimit_unix.go +++ b/common/fdlimit/fdlimit_unix.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build linux darwin netbsd openbsd solaris +// +build linux netbsd openbsd solaris package fdlimit diff --git a/common/fdlimit/fdlimit_windows.go b/common/fdlimit/fdlimit_windows.go index 63a44e0de0..f472153662 100644 --- a/common/fdlimit/fdlimit_windows.go +++ b/common/fdlimit/fdlimit_windows.go @@ -18,6 +18,7 @@ package fdlimit import "fmt" +// hardlimit is the number of file descriptors allowed at max by the kernel. const hardlimit = 16384 // Raise tries to maximize the file descriptor allowance of this process