From 2ac61a99146f2570f753b499b8b1712c2c29974a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 7 Feb 2019 11:59:17 +0100 Subject: [PATCH 01/22] ethapi: default to use eip-155 protected transactions --- internal/ethapi/api.go | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 0aeec8ad10..b3fd4522a9 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -356,11 +356,7 @@ func (s *PrivateAccountAPI) signTransaction(ctx context.Context, args *SendTxArg // Assemble the transaction and sign with the wallet tx := args.toTransaction() - var chainID *big.Int - if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainID - } - return wallet.SignTxWithPassphrase(account, passwd, tx, chainID) + return wallet.SignTxWithPassphrase(account, passwd, tx, s.b.ChainConfig().ChainID) } // SendTransaction will create a transaction from the given arguments and @@ -1186,11 +1182,7 @@ func (s *PublicTransactionPoolAPI) sign(addr common.Address, tx *types.Transacti return nil, err } // Request the wallet to sign the transaction - var chainID *big.Int - if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainID - } - return wallet.SignTx(account, tx, chainID) + return wallet.SignTx(account, tx, s.b.ChainConfig().ChainID) } // SendTxArgs represents the arguments to sumbit a new transaction into the transaction pool. @@ -1306,11 +1298,7 @@ func (s *PublicTransactionPoolAPI) SendTransaction(ctx context.Context, args Sen // Assemble the transaction and sign with the wallet tx := args.toTransaction() - var chainID *big.Int - if config := s.b.ChainConfig(); config.IsEIP155(s.b.CurrentBlock().Number()) { - chainID = config.ChainID - } - signed, err := wallet.SignTx(account, tx, chainID) + signed, err := wallet.SignTx(account, tx, s.b.ChainConfig().ChainID) if err != nil { return common.Hash{}, err } From d6225ab846d1abeeb9ba9e9aad84bc566ddf52f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 5 Feb 2019 12:49:59 +0200 Subject: [PATCH 02/22] cmd/utils, eth: relinquish GC cache to read cache in archive mode --- cmd/utils/flags.go | 4 ++-- common/size.go | 16 ++++++++-------- common/size_test.go | 4 ++-- core/blockchain.go | 4 ++-- core/blockchain_insert.go | 4 ++-- eth/backend.go | 6 ++++++ ethdb/database.go | 3 ++- 7 files changed, 24 insertions(+), 17 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index bf20abe818..66f533102e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -332,12 +332,12 @@ var ( } CacheTrieFlag = cli.IntFlag{ Name: "cache.trie", - Usage: "Percentage of cache memory allowance to use for trie caching", + Usage: "Percentage of cache memory allowance to use for trie caching (default = 25% full mode, 50% archive mode)", Value: 25, } CacheGCFlag = cli.IntFlag{ Name: "cache.gc", - Usage: "Percentage of cache memory allowance to use for trie pruning", + Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", Value: 25, } TrieCacheGenFlag = cli.IntFlag{ diff --git a/common/size.go b/common/size.go index bd0fc85c7d..6381499a48 100644 --- a/common/size.go +++ b/common/size.go @@ -26,10 +26,10 @@ type StorageSize float64 // String implements the stringer interface. func (s StorageSize) String() string { - if s > 1000000 { - return fmt.Sprintf("%.2f mB", s/1000000) - } else if s > 1000 { - return fmt.Sprintf("%.2f kB", s/1000) + if s > 1048576 { + return fmt.Sprintf("%.2f MiB", s/1048576) + } else if s > 1024 { + return fmt.Sprintf("%.2f KiB", s/1024) } else { return fmt.Sprintf("%.2f B", s) } @@ -38,10 +38,10 @@ func (s StorageSize) String() string { // TerminalString implements log.TerminalStringer, formatting a string for console // output during logging. func (s StorageSize) TerminalString() string { - if s > 1000000 { - return fmt.Sprintf("%.2fmB", s/1000000) - } else if s > 1000 { - return fmt.Sprintf("%.2fkB", s/1000) + if s > 1048576 { + return fmt.Sprintf("%.2fMiB", s/1048576) + } else if s > 1024 { + return fmt.Sprintf("%.2fKiB", s/1024) } else { return fmt.Sprintf("%.2fB", s) } diff --git a/common/size_test.go b/common/size_test.go index f5b6c725e2..0938d483c4 100644 --- a/common/size_test.go +++ b/common/size_test.go @@ -25,8 +25,8 @@ func TestStorageSizeString(t *testing.T) { size StorageSize str string }{ - {2381273, "2.38 mB"}, - {2192, "2.19 kB"}, + {2381273, "2.27 MiB"}, + {2192, "2.14 KiB"}, {12, "12.00 B"}, } diff --git a/core/blockchain.go b/core/blockchain.go index 93caf9f369..c852926ef4 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1253,8 +1253,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] stats.processed++ stats.usedGas += usedGas - cache, _ := bc.stateCache.TrieDB().Size() - stats.report(chain, it.index, cache) + dirty, _ := bc.stateCache.TrieDB().Size() + stats.report(chain, it.index, dirty) } // Any blocks remaining here? The only ones we care about are the future ones if block != nil && err == consensus.ErrFutureBlock { diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index cfa32c5aab..f07e24d750 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second // report prints statistics if some number of blocks have been processed // or more than a few seconds have passed since the last message. -func (st *insertStats) report(chain []*types.Block, index int, cache common.StorageSize) { +func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize) { // Fetch the timings for the batch var ( now = mclock.Now() @@ -63,7 +63,7 @@ func (st *insertStats) report(chain []*types.Block, index int, cache common.Stor if timestamp := time.Unix(end.Time().Int64(), 0); time.Since(timestamp) > time.Minute { context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) } - context = append(context, []interface{}{"cache", cache}...) + context = append(context, []interface{}{"dirty", dirty}...) if st.queued > 0 { context = append(context, []interface{}{"queued", st.queued}...) diff --git a/eth/backend.go b/eth/backend.go index 3ec6749b63..6a136182ab 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -113,6 +113,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { log.Warn("Sanitizing invalid miner gas price", "provided", config.MinerGasPrice, "updated", DefaultConfig.MinerGasPrice) config.MinerGasPrice = new(big.Int).Set(DefaultConfig.MinerGasPrice) } + if config.NoPruning && config.TrieDirtyCache > 0 { + config.TrieCleanCache += config.TrieDirtyCache + config.TrieDirtyCache = 0 + } + log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) + // Assemble the Ethereum object chainDb, err := CreateDB(ctx, config, "chaindata") if err != nil { diff --git a/ethdb/database.go b/ethdb/database.go index 6c62d6a386..17f1478e5b 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -25,6 +25,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/syndtr/goleveldb/leveldb" @@ -70,7 +71,7 @@ func NewLDBDatabase(file string, cache int, handles int) (*LDBDatabase, error) { if handles < 16 { handles = 16 } - logger.Info("Allocated cache and file handles", "cache", cache, "handles", handles) + logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) // Open the db and recover any potential corruptions db, err := leveldb.OpenFile(file, &opt.Options{ From 33d0a0efa61fed2b16797fd12161519943943282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 7 Feb 2019 15:46:58 +0100 Subject: [PATCH 03/22] cmd/swarm/global-store: global store cmd (#19014) --- cmd/swarm/config.go | 9 + cmd/swarm/flags.go | 5 + cmd/swarm/global-store/global_store.go | 100 ++++++++++ cmd/swarm/global-store/global_store_test.go | 191 ++++++++++++++++++++ cmd/swarm/global-store/main.go | 104 +++++++++++ cmd/swarm/global-store/run_test.go | 49 +++++ cmd/swarm/main.go | 20 +- swarm/api/config.go | 1 + 8 files changed, 476 insertions(+), 3 deletions(-) create mode 100644 cmd/swarm/global-store/global_store.go create mode 100644 cmd/swarm/global-store/global_store_test.go create mode 100644 cmd/swarm/global-store/main.go create mode 100644 cmd/swarm/global-store/run_test.go diff --git a/cmd/swarm/config.go b/cmd/swarm/config.go index 0203a67984..98d4dee7b9 100644 --- a/cmd/swarm/config.go +++ b/cmd/swarm/config.go @@ -82,6 +82,7 @@ const ( SWARM_ENV_BOOTNODE_MODE = "SWARM_BOOTNODE_MODE" SWARM_ACCESS_PASSWORD = "SWARM_ACCESS_PASSWORD" SWARM_AUTO_DEFAULTPATH = "SWARM_AUTO_DEFAULTPATH" + SWARM_GLOBALSTORE_API = "SWARM_GLOBALSTORE_API" GETH_ENV_DATADIR = "GETH_DATADIR" ) @@ -262,6 +263,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con currentConfig.BootnodeMode = ctx.GlobalBool(SwarmBootnodeModeFlag.Name) } + if ctx.GlobalIsSet(SwarmGlobalStoreAPIFlag.Name) { + currentConfig.GlobalStoreAPI = ctx.GlobalString(SwarmGlobalStoreAPIFlag.Name) + } + return currentConfig } @@ -375,6 +380,10 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) { currentConfig.BootnodeMode = bootnodeMode } + if api := os.Getenv(SWARM_GLOBALSTORE_API); api != "" { + currentConfig.GlobalStoreAPI = api + } + return currentConfig } diff --git a/cmd/swarm/flags.go b/cmd/swarm/flags.go index 4c186cc310..b092a77476 100644 --- a/cmd/swarm/flags.go +++ b/cmd/swarm/flags.go @@ -176,4 +176,9 @@ var ( Name: "user", Usage: "Indicates the user who updates the feed", } + SwarmGlobalStoreAPIFlag = cli.StringFlag{ + Name: "globalstore-api", + Usage: "URL of the Global Store API provider (only for testing)", + EnvVar: SWARM_GLOBALSTORE_API, + } ) diff --git a/cmd/swarm/global-store/global_store.go b/cmd/swarm/global-store/global_store.go new file mode 100644 index 0000000000..a55756e1c4 --- /dev/null +++ b/cmd/swarm/global-store/global_store.go @@ -0,0 +1,100 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "net" + "net/http" + "os" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + "github.com/ethereum/go-ethereum/swarm/storage/mock/db" + "github.com/ethereum/go-ethereum/swarm/storage/mock/mem" + cli "gopkg.in/urfave/cli.v1" +) + +// startHTTP starts a global store with HTTP RPC server. +// It is used for "http" cli command. +func startHTTP(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + log.Info("http", "address", listener.Addr().String()) + + return http.Serve(listener, server) +} + +// startWS starts a global store with WebSocket RPC server. +// It is used for "websocket" cli command. +func startWS(ctx *cli.Context) (err error) { + server, cleanup, err := newServer(ctx) + if err != nil { + return err + } + defer cleanup() + + listener, err := net.Listen("tcp", ctx.String("addr")) + if err != nil { + return err + } + origins := ctx.StringSlice("origins") + log.Info("websocket", "address", listener.Addr().String(), "origins", origins) + + return http.Serve(listener, server.WebsocketHandler(origins)) +} + +// newServer creates a global store and returns its RPC server. +// Returned cleanup function should be called only if err is nil. +func newServer(ctx *cli.Context) (server *rpc.Server, cleanup func(), err error) { + log.PrintOrigins(true) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(ctx.Int("verbosity")), log.StreamHandler(os.Stdout, log.TerminalFormat(false)))) + + cleanup = func() {} + var globalStore mock.GlobalStorer + dir := ctx.String("dir") + if dir != "" { + dbStore, err := db.NewGlobalStore(dir) + if err != nil { + return nil, nil, err + } + cleanup = func() { + dbStore.Close() + } + globalStore = dbStore + log.Info("database global store", "dir", dir) + } else { + globalStore = mem.NewGlobalStore() + log.Info("in-memory global store") + } + + server = rpc.NewServer() + if err := server.RegisterName("mockStore", globalStore); err != nil { + cleanup() + return nil, nil, err + } + + return server, cleanup, nil +} diff --git a/cmd/swarm/global-store/global_store_test.go b/cmd/swarm/global-store/global_store_test.go new file mode 100644 index 0000000000..85f361ed4f --- /dev/null +++ b/cmd/swarm/global-store/global_store_test.go @@ -0,0 +1,191 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rpc" + mockRPC "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" +) + +// TestHTTP_InMemory tests in-memory global store that exposes +// HTTP server. +func TestHTTP_InMemory(t *testing.T) { + testHTTP(t, true) +} + +// TestHTTP_Database tests global store with persisted database +// that exposes HTTP server. +func TestHTTP_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testHTTP(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testHTTP(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with HTTP server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testHTTP(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"http", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + client, err := rpc.DialHTTP("http://" + addr) + if err != nil { + t.Fatal(err) + } + + // wait until global store process is started as + // rpc.DialHTTP is actually not connecting + for i := 0; i < 1000; i++ { + _, err = http.DefaultClient.Get("http://" + addr) + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// TestWebsocket_InMemory tests in-memory global store that exposes +// WebSocket server. +func TestWebsocket_InMemory(t *testing.T) { + testWebsocket(t, true) +} + +// TestWebsocket_Database tests global store with persisted database +// that exposes HTTP server. +func TestWebsocket_Database(t *testing.T) { + dir, err := ioutil.TempDir("", "swarm-global-store-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + // create a fresh global store + testWebsocket(t, true, "--dir", dir) + + // check if data saved by the previous global store instance + testWebsocket(t, false, "--dir", dir) +} + +// testWebsocket starts global store binary with WebSocket server +// and validates that it can store and retrieve data. +// If put is false, no data will be stored, only retrieved, +// giving the possibility to check if data is present in the +// storage directory. +func testWebsocket(t *testing.T, put bool, args ...string) { + addr := findFreeTCPAddress(t) + testCmd := runGlobalStore(t, append([]string{"ws", "--addr", addr}, args...)...) + defer testCmd.Interrupt() + + var client *rpc.Client + var err error + // wait until global store process is started + for i := 0; i < 1000; i++ { + client, err = rpc.DialWebsocket(context.Background(), "ws://"+addr, "") + if err == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + + store := mockRPC.NewGlobalStore(client) + defer store.Close() + + node := store.NewNodeStore(common.HexToAddress("123abc")) + + wantKey := "key" + wantValue := "value" + + if put { + err = node.Put([]byte(wantKey), []byte(wantValue)) + if err != nil { + t.Fatal(err) + } + } + + gotValue, err := node.Get([]byte(wantKey)) + if err != nil { + t.Fatal(err) + } + + if string(gotValue) != wantValue { + t.Errorf("got value %s for key %s, want %s", string(gotValue), wantKey, wantValue) + } +} + +// findFreeTCPAddress returns a local address (IP:Port) to which +// global store can listen on. +func findFreeTCPAddress(t *testing.T) (addr string) { + t.Helper() + + listener, err := net.Listen("tcp", "") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + + return listener.Addr().String() +} diff --git a/cmd/swarm/global-store/main.go b/cmd/swarm/global-store/main.go new file mode 100644 index 0000000000..51df0099ac --- /dev/null +++ b/cmd/swarm/global-store/main.go @@ -0,0 +1,104 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "os" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + cli "gopkg.in/urfave/cli.v1" +) + +var gitCommit string // Git SHA1 commit hash of the release (set via linker flags) + +func main() { + err := newApp().Run(os.Args) + if err != nil { + log.Error(err.Error()) + os.Exit(1) + } +} + +// newApp construct a new instance of Swarm Global Store. +// Method Run is called on it in the main function and in tests. +func newApp() (app *cli.App) { + app = utils.NewApp(gitCommit, "Swarm Global Store") + + app.Name = "global-store" + + // app flags (for all commands) + app.Flags = []cli.Flag{ + cli.IntFlag{ + Name: "verbosity", + Value: 3, + Usage: "verbosity level", + }, + } + + app.Commands = []cli.Command{ + { + Name: "http", + Aliases: []string{"h"}, + Usage: "start swarm global store with http server", + Action: startHTTP, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for http connection", + }, + ), + }, + { + Name: "websocket", + Aliases: []string{"ws"}, + Usage: "start swarm global store with websocket server", + Action: startWS, + // Flags only for "start" command. + // Allow app flags to be specified after the + // command argument. + Flags: append(app.Flags, + cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "data directory", + }, + cli.StringFlag{ + Name: "addr", + Value: "0.0.0.0:3033", + Usage: "address to listen for websocket connection", + }, + cli.StringSliceFlag{ + Name: "origins", + Value: &cli.StringSlice{"*"}, + Usage: "websocket origins", + }, + ), + }, + } + + return app +} diff --git a/cmd/swarm/global-store/run_test.go b/cmd/swarm/global-store/run_test.go new file mode 100644 index 0000000000..d7ef626e55 --- /dev/null +++ b/cmd/swarm/global-store/run_test.go @@ -0,0 +1,49 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func init() { + reexec.Register("swarm-global-store", func() { + if err := newApp().Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) +} + +func runGlobalStore(t *testing.T, args ...string) *cmdtest.TestCmd { + tt := cmdtest.NewTestCmd(t, nil) + tt.Run("swarm-global-store", args...) + return tt +} + +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 839227b2b4..385afad992 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,13 +39,16 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/storage/mock" + mockrpc "github.com/ethereum/go-ethereum/swarm/storage/mock/rpc" "github.com/ethereum/go-ethereum/swarm/tracing" sv "github.com/ethereum/go-ethereum/swarm/version" - "gopkg.in/urfave/cli.v1" + cli "gopkg.in/urfave/cli.v1" ) const clientIdentifier = "swarm" @@ -196,6 +199,7 @@ func init() { SwarmStorePath, SwarmStoreCapacity, SwarmStoreCacheCapacity, + SwarmGlobalStoreAPIFlag, } rpcFlags := []cli.Flag{ utils.WSEnabledFlag, @@ -325,8 +329,18 @@ func bzzd(ctx *cli.Context) error { func registerBzzService(bzzconfig *bzzapi.Config, stack *node.Node) { //define the swarm service boot function boot := func(_ *node.ServiceContext) (node.Service, error) { - // In production, mockStore must be always nil. - return swarm.NewSwarm(bzzconfig, nil) + var nodeStore *mock.NodeStore + if bzzconfig.GlobalStoreAPI != "" { + // connect to global store + client, err := rpc.Dial(bzzconfig.GlobalStoreAPI) + if err != nil { + return nil, fmt.Errorf("global store: %v", err) + } + globalStore := mockrpc.NewGlobalStore(client) + // create a node store for this swarm key on global store + nodeStore = globalStore.NewNodeStore(common.HexToAddress(bzzconfig.BzzKey)) + } + return swarm.NewSwarm(bzzconfig, nodeStore) } //register within the ethereum node if err := stack.Register(boot); err != nil { diff --git a/swarm/api/config.go b/swarm/api/config.go index 54dd67ba83..b8de16f5f0 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -71,6 +71,7 @@ type Config struct { SwapAPI string Cors string BzzAccount string + GlobalStoreAPI string privateKey *ecdsa.PrivateKey } From 41597c2856d6ac7328baca1340c3e36ab0edd382 Mon Sep 17 00:00:00 2001 From: holisticode Date: Thu, 7 Feb 2019 09:49:19 -0500 Subject: [PATCH 04/22] swarm: Debug API and HasChunks() API endpoint (#18980) --- cmd/swarm/main.go | 2 +- swarm/api/inspector.go | 58 +++++++++++++++++++++++++++++ swarm/api/testapi.go | 34 ----------------- swarm/network/stream/common_test.go | 5 +++ swarm/storage/common_test.go | 9 +++++ swarm/storage/ldbstore.go | 12 ++++++ swarm/storage/localstore.go | 7 ++++ swarm/storage/localstore_test.go | 33 ++++++++++++++++ swarm/storage/memstore.go | 5 +++ swarm/storage/netstore.go | 7 ++++ swarm/storage/types.go | 8 +++- swarm/swarm.go | 2 +- 12 files changed, 145 insertions(+), 37 deletions(-) create mode 100644 swarm/api/inspector.go delete mode 100644 swarm/api/testapi.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 385afad992..af00503ea3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -468,5 +468,5 @@ func setSwarmBootstrapNodes(ctx *cli.Context, cfg *node.Config) { } cfg.P2P.BootstrapNodes = append(cfg.P2P.BootstrapNodes, node) } - log.Debug("added default swarm bootnodes", "length", len(cfg.P2P.BootstrapNodes)) + } diff --git a/swarm/api/inspector.go b/swarm/api/inspector.go new file mode 100644 index 0000000000..6706b32e65 --- /dev/null +++ b/swarm/api/inspector.go @@ -0,0 +1,58 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package api + +import ( + "context" + + "github.com/ethereum/go-ethereum/swarm/network" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +type Inspector struct { + api *API + hive *network.Hive + netStore *storage.NetStore +} + +func NewInspector(api *API, hive *network.Hive, netStore *storage.NetStore) *Inspector { + return &Inspector{api, hive, netStore} +} + +// Hive prints the kademlia table +func (inspector *Inspector) Hive() string { + return inspector.hive.String() +} + +type HasInfo struct { + Addr string `json:"address"` + Has bool `json:"has"` +} + +// Has checks whether each chunk address is present in the underlying datastore, +// the bool in the returned structs indicates if the underlying datastore has +// the chunk stored with the given address (true), or not (false) +func (inspector *Inspector) Has(chunkAddresses []storage.Address) []HasInfo { + results := make([]HasInfo, 0) + for _, addr := range chunkAddresses { + res := HasInfo{} + res.Addr = addr.String() + res.Has = inspector.netStore.Has(context.Background(), addr) + results = append(results, res) + } + return results +} diff --git a/swarm/api/testapi.go b/swarm/api/testapi.go deleted file mode 100644 index 6fec55f559..0000000000 --- a/swarm/api/testapi.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package api - -import ( - "github.com/ethereum/go-ethereum/swarm/network" -) - -type Control struct { - api *API - hive *network.Hive -} - -func NewControl(api *API, hive *network.Hive) *Control { - return &Control{api, hive} -} - -func (c *Control) Hive() string { - return c.hive.String() -} diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 3b6e4a9465..8a7d851fb7 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -214,6 +214,11 @@ func newRoundRobinStore(stores ...storage.ChunkStore) *roundRobinStore { } } +// not used in this context, only to fulfill ChunkStore interface +func (rrs *roundRobinStore) Has(ctx context.Context, addr storage.Address) bool { + panic("RoundRobinStor doesn't support HasChunk") +} + func (rrs *roundRobinStore) Get(ctx context.Context, addr storage.Address) (storage.Chunk, error) { return nil, errors.New("get not well defined on round robin store") } diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 6c737af44e..23d43beccf 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -267,6 +267,15 @@ func (m *MapChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { return chunk, nil } +// Need to implement Has from SyncChunkStore +func (m *MapChunkStore) Has(ctx context.Context, ref Address) bool { + m.mu.RLock() + defer m.mu.RUnlock() + + _, has := m.chunks[ref.Hex()] + return has +} + func (m *MapChunkStore) Close() { } diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 635d334297..a2f24eff06 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -969,6 +969,18 @@ func (s *LDBStore) Get(_ context.Context, addr Address) (chunk Chunk, err error) return s.get(addr) } +// Has queries the underlying DB if a chunk with the given address is stored +// Returns true if the chunk is found, false if not +func (s *LDBStore) Has(_ context.Context, addr Address) bool { + s.lock.RLock() + defer s.lock.RUnlock() + + ikey := getIndexKey(addr) + _, err := s.db.Get(ikey) + + return err == nil +} + // TODO: To conform with other private methods of this object indices should not be updated func (s *LDBStore) get(addr Address) (chunk *chunk, err error) { if s.closed { diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 956560902b..eefb7565a5 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -132,6 +132,13 @@ func (ls *LocalStore) Put(ctx context.Context, chunk Chunk) error { return err } +// Has queries the underlying DbStore if a chunk with the given address +// is being stored there. +// Returns true if it is stored, false if not +func (ls *LocalStore) Has(ctx context.Context, addr Address) bool { + return ls.DbStore.Has(ctx, addr) +} + // Get(chunk *Chunk) looks up a chunk in the local stores // This method is blocking until the chunk is retrieved // so additional timeout may be needed to wrap this call if diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 7a4162a475..ec69951c4f 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -209,3 +209,36 @@ func setupLocalStore(t *testing.T, ldbCap int) (ls *LocalStore, cleanup func()) return store, cleanup } + +func TestHas(t *testing.T) { + ldbCap := defaultGCRatio + store, cleanup := setupLocalStore(t, ldbCap) + defer cleanup() + + nonStoredAddr := GenerateRandomChunk(128).Address() + + has := store.Has(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected Has() to return false, but returned true!") + } + + storeChunks := GenerateRandomChunks(128, 3) + for _, ch := range storeChunks { + err := store.Put(context.Background(), ch) + if err != nil { + t.Fatalf("Expected store to store chunk, but it failed: %v", err) + } + + has := store.Has(context.Background(), ch.Address()) + if !has { + t.Fatal("Expected Has() to return true, but returned false!") + } + } + + //let's be paranoic and test again that the non-existent chunk returns false + has = store.Has(context.Background(), nonStoredAddr) + if has { + t.Fatal("Expected Has() to return false, but returned true!") + } + +} diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 86e5813d1b..611ac3bc51 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -48,6 +48,11 @@ func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) { } } +// Has needed to implement SyncChunkStore +func (m *MemStore) Has(_ context.Context, addr Address) bool { + return m.cache.Contains(addr) +} + func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if m.disabled { return nil, ErrChunkNotFound diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index 16bc48a9a2..b24d08bc2d 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -158,6 +158,13 @@ func (n *NetStore) get(ctx context.Context, ref Address) (Chunk, func(context.Co return chunk, nil, nil } +// Has is the storage layer entry point to query the underlying +// database to return if it has a chunk or not. +// Called from the DebugAPI +func (n *NetStore) Has(ctx context.Context, ref Address) bool { + return n.store.Has(ctx, ref) +} + // getOrCreateFetcher attempts at retrieving an existing fetchers // if none exists, creates one and saves it in the fetchers cache // caller must hold the lock diff --git a/swarm/storage/types.go b/swarm/storage/types.go index d792352252..7ec21328e2 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -292,6 +292,7 @@ func (v *ContentAddressValidator) Validate(chunk Chunk) bool { type ChunkStore interface { Put(ctx context.Context, ch Chunk) (err error) Get(rctx context.Context, ref Address) (ch Chunk, err error) + Has(rctx context.Context, ref Address) bool Close() } @@ -314,7 +315,12 @@ func (f *FakeChunkStore) Put(_ context.Context, ch Chunk) error { return nil } -// Gut doesn't store anything it is just here to implement ChunkStore +// Has doesn't do anything it is just here to implement ChunkStore +func (f *FakeChunkStore) Has(_ context.Context, ref Address) bool { + panic("FakeChunkStore doesn't support HasChunk") +} + +// Get doesn't store anything it is just here to implement ChunkStore func (f *FakeChunkStore) Get(_ context.Context, ref Address) (Chunk, error) { panic("FakeChunkStore doesn't support Get") } diff --git a/swarm/swarm.go b/swarm/swarm.go index 8274a168d6..705fc43975 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -485,7 +485,7 @@ func (self *Swarm) APIs() []rpc.API { { Namespace: "bzz", Version: "3.0", - Service: api.NewControl(self.api, self.bzz.Hive), + Service: api.NewInspector(self.api, self.bzz.Hive, self.netStore), Public: false, }, { From 4f3d22f06c546f36487b33dfb6b5cb4df3ecf073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Thu, 7 Feb 2019 18:40:26 +0100 Subject: [PATCH 05/22] swarm/storage/localstore: new localstore package (#19015) --- swarm/storage/localstore/doc.go | 56 ++ swarm/storage/localstore/gc.go | 302 ++++++++++ swarm/storage/localstore/gc_test.go | 358 ++++++++++++ swarm/storage/localstore/index_test.go | 227 ++++++++ swarm/storage/localstore/localstore.go | 431 +++++++++++++++ swarm/storage/localstore/localstore_test.go | 520 ++++++++++++++++++ swarm/storage/localstore/mode_get.go | 154 ++++++ swarm/storage/localstore/mode_get_test.go | 237 ++++++++ swarm/storage/localstore/mode_put.go | 160 ++++++ swarm/storage/localstore/mode_put_test.go | 300 ++++++++++ swarm/storage/localstore/mode_set.go | 205 +++++++ swarm/storage/localstore/mode_set_test.go | 128 +++++ .../localstore/retrieval_index_test.go | 150 +++++ swarm/storage/localstore/subscription_pull.go | 193 +++++++ .../localstore/subscription_pull_test.go | 478 ++++++++++++++++ swarm/storage/localstore/subscription_push.go | 145 +++++ .../localstore/subscription_push_test.go | 200 +++++++ 17 files changed, 4244 insertions(+) create mode 100644 swarm/storage/localstore/doc.go create mode 100644 swarm/storage/localstore/gc.go create mode 100644 swarm/storage/localstore/gc_test.go create mode 100644 swarm/storage/localstore/index_test.go create mode 100644 swarm/storage/localstore/localstore.go create mode 100644 swarm/storage/localstore/localstore_test.go create mode 100644 swarm/storage/localstore/mode_get.go create mode 100644 swarm/storage/localstore/mode_get_test.go create mode 100644 swarm/storage/localstore/mode_put.go create mode 100644 swarm/storage/localstore/mode_put_test.go create mode 100644 swarm/storage/localstore/mode_set.go create mode 100644 swarm/storage/localstore/mode_set_test.go create mode 100644 swarm/storage/localstore/retrieval_index_test.go create mode 100644 swarm/storage/localstore/subscription_pull.go create mode 100644 swarm/storage/localstore/subscription_pull_test.go create mode 100644 swarm/storage/localstore/subscription_push.go create mode 100644 swarm/storage/localstore/subscription_push_test.go diff --git a/swarm/storage/localstore/doc.go b/swarm/storage/localstore/doc.go new file mode 100644 index 0000000000..98f6fc40aa --- /dev/null +++ b/swarm/storage/localstore/doc.go @@ -0,0 +1,56 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Package localstore provides disk storage layer for Swarm Chunk persistence. +It uses swarm/shed abstractions on top of github.com/syndtr/goleveldb LevelDB +implementation. + +The main type is DB which manages the storage by providing methods to +access and add Chunks and to manage their status. + +Modes are abstractions that do specific changes to Chunks. There are three +mode types: + + - ModeGet, for Chunk access + - ModePut, for adding Chunks to the database + - ModeSet, for changing Chunk statuses + +Every mode type has a corresponding type (Getter, Putter and Setter) +that provides adequate method to perform the opperation and that type +should be injected into localstore consumers instead the whole DB. +This provides more clear insight which operations consumer is performing +on the database. + +Getters, Putters and Setters accept different get, put and set modes +to perform different actions. For example, ModeGet has two different +variables ModeGetRequest and ModeGetSync and two different Getters +can be constructed with them that are used when the chunk is requested +or when the chunk is synced as this two events are differently changing +the database. + +Subscription methods are implemented for a specific purpose of +continuous iterations over Chunks that should be provided to +Push and Pull syncing. + +DB implements an internal garbage collector that removes only synced +Chunks from the database based on their most recent access time. + +Internally, DB stores Chunk data and any required information, such as +store and access timestamps in different shed indexes that can be +iterated on by garbage collector or subscriptions. +*/ +package localstore diff --git a/swarm/storage/localstore/gc.go b/swarm/storage/localstore/gc.go new file mode 100644 index 0000000000..7718d1e589 --- /dev/null +++ b/swarm/storage/localstore/gc.go @@ -0,0 +1,302 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +/* +Counting number of items in garbage collection index + +The number of items in garbage collection index is not the same as the number of +chunks in retrieval index (total number of stored chunks). Chunk can be garbage +collected only when it is set to a synced state by ModSetSync, and only then can +be counted into garbage collection size, which determines whether a number of +chunk should be removed from the storage by the garbage collection. This opens a +possibility that the storage size exceeds the limit if files are locally +uploaded and the node is not connected to other nodes or there is a problem with +syncing. + +Tracking of garbage collection size (gcSize) is focused on performance. Key +points: + + 1. counting the number of key/value pairs in LevelDB takes around 0.7s for 1e6 + on a very fast ssd (unacceptable long time in reality) + 2. locking leveldb batch writes with a global mutex (serial batch writes) is + not acceptable, we should use locking per chunk address + +Because of point 1. we cannot count the number of items in garbage collection +index in New constructor as it could last very long for realistic scenarios +where limit is 5e6 and nodes are running on slower hdd disks or cloud providers +with low IOPS. + +Point 2. is a performance optimization to allow parallel batch writes with +getters, putters and setters. Every single batch that they create contain only +information related to a single chunk, no relations with other chunks or shared +statistical data (like gcSize). This approach avoids race conditions on writing +batches in parallel, but creates a problem of synchronizing statistical data +values like gcSize. With global mutex lock, any data could be written by any +batch, but would not use utilize the full potential of leveldb parallel writes. + +To mitigate this two problems, the implementation of counting and persisting +gcSize is split into two parts. One is the in-memory value (gcSize) that is fast +to read and write with a dedicated mutex (gcSizeMu) if the batch which adds or +removes items from garbage collection index is successful. The second part is +the reliable persistence of this value to leveldb database, as storedGCSize +field. This database field is saved by writeGCSizeWorker and writeGCSize +functions when in-memory gcSize variable is changed, but no too often to avoid +very frequent database writes. This database writes are triggered by +writeGCSizeTrigger when a call is made to function incGCSize. Trigger ensures +that no database writes are done only when gcSize is changed (contrary to a +simpler periodic writes or checks). A backoff of 10s in writeGCSizeWorker +ensures that no frequent batch writes are made. Saving the storedGCSize on +database Close function ensures that in-memory gcSize is persisted when database +is closed. + +This persistence must be resilient to failures like panics. For this purpose, a +collection of hashes that are added to the garbage collection index, but still +not persisted to storedGCSize, must be tracked to count them in when DB is +constructed again with New function after the failure (swarm node restarts). On +every batch write that adds a new item to garbage collection index, the same +hash is added to gcUncountedHashesIndex. This ensures that there is a persisted +information which hashes were added to the garbage collection index. But, when +the storedGCSize is saved by writeGCSize function, this values are removed in +the same batch in which storedGCSize is changed to ensure consistency. When the +panic happen, or database Close method is not saved. The database storage +contains all information to reliably and efficiently get the correct number of +items in garbage collection index. This is performed in the New function when +all hashes in gcUncountedHashesIndex are counted, added to the storedGCSize and +saved to the disk before the database is constructed again. Index +gcUncountedHashesIndex is acting as dirty bit for recovery that provides +information what needs to be corrected. With a simple dirty bit, the whole +garbage collection index should me counted on recovery instead only the items in +gcUncountedHashesIndex. Because of the triggering mechanizm of writeGCSizeWorker +and relatively short backoff time, the number of hashes in +gcUncountedHashesIndex should be low and it should take a very short time to +recover from the previous failure. If there was no failure and +gcUncountedHashesIndex is empty, which is the usual case, New function will take +the minimal time to return. +*/ + +package localstore + +import ( + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/syndtr/goleveldb/leveldb" +) + +var ( + // gcTargetRatio defines the target number of items + // in garbage collection index that will not be removed + // on garbage collection. The target number of items + // is calculated by gcTarget function. This value must be + // in range (0,1]. For example, with 0.9 value, + // garbage collection will leave 90% of defined capacity + // in database after its run. This prevents frequent + // garbage collection runs. + gcTargetRatio = 0.9 + // gcBatchSize limits the number of chunks in a single + // leveldb batch on garbage collection. + gcBatchSize int64 = 1000 +) + +// collectGarbageWorker is a long running function that waits for +// collectGarbageTrigger channel to signal a garbage collection +// run. GC run iterates on gcIndex and removes older items +// form retrieval and other indexes. +func (db *DB) collectGarbageWorker() { + for { + select { + case <-db.collectGarbageTrigger: + // run a single collect garbage run and + // if done is false, gcBatchSize is reached and + // another collect garbage run is needed + collectedCount, done, err := db.collectGarbage() + if err != nil { + log.Error("localstore collect garbage", "err", err) + } + // check if another gc run is needed + if !done { + db.triggerGarbageCollection() + } + + if testHookCollectGarbage != nil { + testHookCollectGarbage(collectedCount) + } + case <-db.close: + return + } + } +} + +// collectGarbage removes chunks from retrieval and other +// indexes if maximal number of chunks in database is reached. +// This function returns the number of removed chunks. If done +// is false, another call to this function is needed to collect +// the rest of the garbage as the batch size limit is reached. +// This function is called in collectGarbageWorker. +func (db *DB) collectGarbage() (collectedCount int64, done bool, err error) { + batch := new(leveldb.Batch) + target := db.gcTarget() + + done = true + err = db.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) { + // protect parallel updates + unlock, err := db.lockAddr(item.Address) + if err != nil { + return false, err + } + defer unlock() + + gcSize := db.getGCSize() + if gcSize-collectedCount <= target { + return true, nil + } + // delete from retrieve, pull, gc + db.retrievalDataIndex.DeleteInBatch(batch, item) + db.retrievalAccessIndex.DeleteInBatch(batch, item) + db.pullIndex.DeleteInBatch(batch, item) + db.gcIndex.DeleteInBatch(batch, item) + collectedCount++ + if collectedCount >= gcBatchSize { + // bach size limit reached, + // another gc run is needed + done = false + return true, nil + } + return false, nil + }, nil) + if err != nil { + return 0, false, err + } + + err = db.shed.WriteBatch(batch) + if err != nil { + return 0, false, err + } + // batch is written, decrement gcSize + db.incGCSize(-collectedCount) + return collectedCount, done, nil +} + +// gcTrigger retruns the absolute value for garbage collection +// target value, calculated from db.capacity and gcTargetRatio. +func (db *DB) gcTarget() (target int64) { + return int64(float64(db.capacity) * gcTargetRatio) +} + +// incGCSize increments gcSize by the provided number. +// If count is negative, it will decrement gcSize. +func (db *DB) incGCSize(count int64) { + if count == 0 { + return + } + + db.gcSizeMu.Lock() + new := db.gcSize + count + db.gcSize = new + db.gcSizeMu.Unlock() + + select { + case db.writeGCSizeTrigger <- struct{}{}: + default: + } + if new >= db.capacity { + db.triggerGarbageCollection() + } +} + +// getGCSize returns gcSize value by locking it +// with gcSizeMu mutex. +func (db *DB) getGCSize() (count int64) { + db.gcSizeMu.RLock() + count = db.gcSize + db.gcSizeMu.RUnlock() + return count +} + +// triggerGarbageCollection signals collectGarbageWorker +// to call collectGarbage. +func (db *DB) triggerGarbageCollection() { + select { + case db.collectGarbageTrigger <- struct{}{}: + case <-db.close: + default: + } +} + +// writeGCSizeWorker writes gcSize on trigger event +// and waits writeGCSizeDelay after each write. +// It implements a linear backoff with delay of +// writeGCSizeDelay duration to avoid very frequent +// database operations. +func (db *DB) writeGCSizeWorker() { + for { + select { + case <-db.writeGCSizeTrigger: + err := db.writeGCSize(db.getGCSize()) + if err != nil { + log.Error("localstore write gc size", "err", err) + } + // Wait some time before writing gc size in the next + // iteration. This prevents frequent I/O operations. + select { + case <-time.After(10 * time.Second): + case <-db.close: + return + } + case <-db.close: + return + } + } +} + +// writeGCSize stores the number of items in gcIndex. +// It removes all hashes from gcUncountedHashesIndex +// not to include them on the next DB initialization +// (New function) when gcSize is counted. +func (db *DB) writeGCSize(gcSize int64) (err error) { + const maxBatchSize = 1000 + + batch := new(leveldb.Batch) + db.storedGCSize.PutInBatch(batch, uint64(gcSize)) + batchSize := 1 + + // use only one iterator as it acquires its snapshot + // not to remove hashes from index that are added + // after stored gc size is written + err = db.gcUncountedHashesIndex.Iterate(func(item shed.Item) (stop bool, err error) { + db.gcUncountedHashesIndex.DeleteInBatch(batch, item) + batchSize++ + if batchSize >= maxBatchSize { + err = db.shed.WriteBatch(batch) + if err != nil { + return false, err + } + batch.Reset() + batchSize = 0 + } + return false, nil + }, nil) + if err != nil { + return err + } + return db.shed.WriteBatch(batch) +} + +// testHookCollectGarbage is a hook that can provide +// information when a garbage collection run is done +// and how many items it removed. +var testHookCollectGarbage func(collectedCount int64) diff --git a/swarm/storage/localstore/gc_test.go b/swarm/storage/localstore/gc_test.go new file mode 100644 index 0000000000..eb039a554a --- /dev/null +++ b/swarm/storage/localstore/gc_test.go @@ -0,0 +1,358 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "io/ioutil" + "math/rand" + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TestDB_collectGarbageWorker tests garbage collection runs +// by uploading and syncing a number of chunks. +func TestDB_collectGarbageWorker(t *testing.T) { + testDB_collectGarbageWorker(t) +} + +// TestDB_collectGarbageWorker_multipleBatches tests garbage +// collection runs by uploading and syncing a number of +// chunks by having multiple smaller batches. +func TestDB_collectGarbageWorker_multipleBatches(t *testing.T) { + // lower the maximal number of chunks in a single + // gc batch to ensure multiple batches. + defer func(s int64) { gcBatchSize = s }(gcBatchSize) + gcBatchSize = 2 + + testDB_collectGarbageWorker(t) +} + +// testDB_collectGarbageWorker is a helper test function to test +// garbage collection runs by uploading and syncing a number of chunks. +func testDB_collectGarbageWorker(t *testing.T) { + chunkCount := 150 + + testHookCollectGarbageChan := make(chan int64) + defer setTestHookCollectGarbage(func(collectedCount int64) { + testHookCollectGarbageChan <- collectedCount + })() + + db, cleanupFunc := newTestDB(t, &Options{ + Capacity: 100, + }) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + syncer := db.NewSetter(ModeSetSync) + + addrs := make([]storage.Address, 0) + + // upload random chunks + for i := 0; i < chunkCount; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + err = syncer.Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + addrs = append(addrs, chunk.Address()) + } + + gcTarget := db.gcTarget() + + for { + select { + case <-testHookCollectGarbageChan: + case <-time.After(10 * time.Second): + t.Error("collect garbage timeout") + } + gcSize := db.getGCSize() + if gcSize == gcTarget { + break + } + } + + t.Run("pull index count", newItemsCountTest(db.pullIndex, int(gcTarget))) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, int(gcTarget))) + + t.Run("gc size", newIndexGCSizeTest(db)) + + // the first synced chunk should be removed + t.Run("get the first synced chunk", func(t *testing.T) { + _, err := db.NewGetter(ModeGetRequest).Get(addrs[0]) + if err != storage.ErrChunkNotFound { + t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound) + } + }) + + // last synced chunk should not be removed + t.Run("get most recent synced chunk", func(t *testing.T) { + _, err := db.NewGetter(ModeGetRequest).Get(addrs[len(addrs)-1]) + if err != nil { + t.Fatal(err) + } + }) + + // cleanup: drain the last testHookCollectGarbageChan + // element before calling deferred functions not to block + // collectGarbageWorker loop, preventing the race in + // setting testHookCollectGarbage function + select { + case <-testHookCollectGarbageChan: + default: + } +} + +// TestDB_collectGarbageWorker_withRequests is a helper test function +// to test garbage collection runs by uploading, syncing and +// requesting a number of chunks. +func TestDB_collectGarbageWorker_withRequests(t *testing.T) { + db, cleanupFunc := newTestDB(t, &Options{ + Capacity: 100, + }) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + syncer := db.NewSetter(ModeSetSync) + + testHookCollectGarbageChan := make(chan int64) + defer setTestHookCollectGarbage(func(collectedCount int64) { + testHookCollectGarbageChan <- collectedCount + })() + + addrs := make([]storage.Address, 0) + + // upload random chunks just up to the capacity + for i := 0; i < int(db.capacity)-1; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + err = syncer.Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + addrs = append(addrs, chunk.Address()) + } + + // request the latest synced chunk + // to prioritize it in the gc index + // not to be collected + _, err := db.NewGetter(ModeGetRequest).Get(addrs[0]) + if err != nil { + t.Fatal(err) + } + + // upload and sync another chunk to trigger + // garbage collection + chunk := generateRandomChunk() + err = uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + err = syncer.Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + addrs = append(addrs, chunk.Address()) + + // wait for garbage collection + + gcTarget := db.gcTarget() + + var totalCollectedCount int64 + for { + select { + case c := <-testHookCollectGarbageChan: + totalCollectedCount += c + case <-time.After(10 * time.Second): + t.Error("collect garbage timeout") + } + gcSize := db.getGCSize() + if gcSize == gcTarget { + break + } + } + + wantTotalCollectedCount := int64(len(addrs)) - gcTarget + if totalCollectedCount != wantTotalCollectedCount { + t.Errorf("total collected chunks %v, want %v", totalCollectedCount, wantTotalCollectedCount) + } + + t.Run("pull index count", newItemsCountTest(db.pullIndex, int(gcTarget))) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, int(gcTarget))) + + t.Run("gc size", newIndexGCSizeTest(db)) + + // requested chunk should not be removed + t.Run("get requested chunk", func(t *testing.T) { + _, err := db.NewGetter(ModeGetRequest).Get(addrs[0]) + if err != nil { + t.Fatal(err) + } + }) + + // the second synced chunk should be removed + t.Run("get gc-ed chunk", func(t *testing.T) { + _, err := db.NewGetter(ModeGetRequest).Get(addrs[1]) + if err != storage.ErrChunkNotFound { + t.Errorf("got error %v, want %v", err, storage.ErrChunkNotFound) + } + }) + + // last synced chunk should not be removed + t.Run("get most recent synced chunk", func(t *testing.T) { + _, err := db.NewGetter(ModeGetRequest).Get(addrs[len(addrs)-1]) + if err != nil { + t.Fatal(err) + } + }) +} + +// TestDB_gcSize checks if gcSize has a correct value after +// database is initialized with existing data. +func TestDB_gcSize(t *testing.T) { + dir, err := ioutil.TempDir("", "localstore-stored-gc-size") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + baseKey := make([]byte, 32) + if _, err := rand.Read(baseKey); err != nil { + t.Fatal(err) + } + db, err := New(dir, baseKey, nil) + if err != nil { + t.Fatal(err) + } + + uploader := db.NewPutter(ModePutUpload) + syncer := db.NewSetter(ModeSetSync) + + count := 100 + + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + err = syncer.Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + } + + // DB.Close writes gc size to disk, so + // Instead calling Close, simulate database shutdown + // without it. + close(db.close) + db.updateGCWG.Wait() + err = db.shed.Close() + if err != nil { + t.Fatal(err) + } + + db, err = New(dir, baseKey, nil) + if err != nil { + t.Fatal(err) + } + + t.Run("gc index size", newIndexGCSizeTest(db)) + + t.Run("gc uncounted hashes index count", newItemsCountTest(db.gcUncountedHashesIndex, 0)) +} + +// setTestHookCollectGarbage sets testHookCollectGarbage and +// returns a function that will reset it to the +// value before the change. +func setTestHookCollectGarbage(h func(collectedCount int64)) (reset func()) { + current := testHookCollectGarbage + reset = func() { testHookCollectGarbage = current } + testHookCollectGarbage = h + return reset +} + +// TestSetTestHookCollectGarbage tests if setTestHookCollectGarbage changes +// testHookCollectGarbage function correctly and if its reset function +// resets the original function. +func TestSetTestHookCollectGarbage(t *testing.T) { + // Set the current function after the test finishes. + defer func(h func(collectedCount int64)) { testHookCollectGarbage = h }(testHookCollectGarbage) + + // expected value for the unchanged function + original := 1 + // expected value for the changed function + changed := 2 + + // this variable will be set with two different functions + var got int + + // define the original (unchanged) functions + testHookCollectGarbage = func(_ int64) { + got = original + } + + // set got variable + testHookCollectGarbage(0) + + // test if got variable is set correctly + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } + + // set the new function + reset := setTestHookCollectGarbage(func(_ int64) { + got = changed + }) + + // set got variable + testHookCollectGarbage(0) + + // test if got variable is set correctly to changed value + if got != changed { + t.Errorf("got hook value %v, want %v", got, changed) + } + + // set the function to the original one + reset() + + // set got variable + testHookCollectGarbage(0) + + // test if got variable is set correctly to original value + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } +} diff --git a/swarm/storage/localstore/index_test.go b/swarm/storage/localstore/index_test.go new file mode 100644 index 0000000000..d9abf440f1 --- /dev/null +++ b/swarm/storage/localstore/index_test.go @@ -0,0 +1,227 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TestDB_pullIndex validates the ordering of keys in pull index. +// Pull index key contains PO prefix which is calculated from +// DB base key and chunk address. This is not an Item field +// which are checked in Mode tests. +// This test uploads chunks, sorts them in expected order and +// validates that pull index iterator will iterate it the same +// order. +func TestDB_pullIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + chunkCount := 50 + + chunks := make([]testIndexChunk, chunkCount) + + // upload random chunks + for i := 0; i < chunkCount; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + chunks[i] = testIndexChunk{ + Chunk: chunk, + // this timestamp is not the same as in + // the index, but given that uploads + // are sequential and that only ordering + // of events matter, this information is + // sufficient + storeTimestamp: now(), + } + } + + testItemsOrder(t, db.pullIndex, chunks, func(i, j int) (less bool) { + poi := storage.Proximity(db.baseKey, chunks[i].Address()) + poj := storage.Proximity(db.baseKey, chunks[j].Address()) + if poi < poj { + return true + } + if poi > poj { + return false + } + if chunks[i].storeTimestamp < chunks[j].storeTimestamp { + return true + } + if chunks[i].storeTimestamp > chunks[j].storeTimestamp { + return false + } + return bytes.Compare(chunks[i].Address(), chunks[j].Address()) == -1 + }) +} + +// TestDB_gcIndex validates garbage collection index by uploading +// a chunk with and performing operations using synced, access and +// request modes. +func TestDB_gcIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + chunkCount := 50 + + chunks := make([]testIndexChunk, chunkCount) + + // upload random chunks + for i := 0; i < chunkCount; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + chunks[i] = testIndexChunk{ + Chunk: chunk, + } + } + + // check if all chunks are stored + newItemsCountTest(db.pullIndex, chunkCount)(t) + + // check that chunks are not collectable for garbage + newItemsCountTest(db.gcIndex, 0)(t) + + // set update gc test hook to signal when + // update gc goroutine is done by sending to + // testHookUpdateGCChan channel, which is + // used to wait for indexes change verifications + testHookUpdateGCChan := make(chan struct{}) + defer setTestHookUpdateGC(func() { + testHookUpdateGCChan <- struct{}{} + })() + + t.Run("request unsynced", func(t *testing.T) { + chunk := chunks[1] + + _, err := db.NewGetter(ModeGetRequest).Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + + // the chunk is not synced + // should not be in the garbace collection index + newItemsCountTest(db.gcIndex, 0)(t) + + newIndexGCSizeTest(db)(t) + }) + + t.Run("sync one chunk", func(t *testing.T) { + chunk := chunks[0] + + err := db.NewSetter(ModeSetSync).Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + // the chunk is synced and should be in gc index + newItemsCountTest(db.gcIndex, 1)(t) + + newIndexGCSizeTest(db)(t) + }) + + t.Run("sync all chunks", func(t *testing.T) { + setter := db.NewSetter(ModeSetSync) + + for i := range chunks { + err := setter.Set(chunks[i].Address()) + if err != nil { + t.Fatal(err) + } + } + + testItemsOrder(t, db.gcIndex, chunks, nil) + + newIndexGCSizeTest(db)(t) + }) + + t.Run("request one chunk", func(t *testing.T) { + i := 6 + + _, err := db.NewGetter(ModeGetRequest).Get(chunks[i].Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + + // move the chunk to the end of the expected gc + c := chunks[i] + chunks = append(chunks[:i], chunks[i+1:]...) + chunks = append(chunks, c) + + testItemsOrder(t, db.gcIndex, chunks, nil) + + newIndexGCSizeTest(db)(t) + }) + + t.Run("random chunk request", func(t *testing.T) { + requester := db.NewGetter(ModeGetRequest) + + rand.Shuffle(len(chunks), func(i, j int) { + chunks[i], chunks[j] = chunks[j], chunks[i] + }) + + for _, chunk := range chunks { + _, err := requester.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + } + + testItemsOrder(t, db.gcIndex, chunks, nil) + + newIndexGCSizeTest(db)(t) + }) + + t.Run("remove one chunk", func(t *testing.T) { + i := 3 + + err := db.NewSetter(modeSetRemove).Set(chunks[i].Address()) + if err != nil { + t.Fatal(err) + } + + // remove the chunk from the expected chunks in gc index + chunks = append(chunks[:i], chunks[i+1:]...) + + testItemsOrder(t, db.gcIndex, chunks, nil) + + newIndexGCSizeTest(db)(t) + }) +} diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go new file mode 100644 index 0000000000..7a9fb54f55 --- /dev/null +++ b/swarm/storage/localstore/localstore.go @@ -0,0 +1,431 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "encoding/binary" + "encoding/hex" + "errors" + "sync" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/storage/mock" +) + +var ( + // ErrInvalidMode is retuned when an unknown Mode + // is provided to the function. + ErrInvalidMode = errors.New("invalid mode") + // ErrAddressLockTimeout is returned when the same chunk + // is updated in parallel and one of the updates + // takes longer then the configured timeout duration. + ErrAddressLockTimeout = errors.New("address lock timeout") +) + +var ( + // Default value for Capacity DB option. + defaultCapacity int64 = 5000000 + // Limit the number of goroutines created by Getters + // that call updateGC function. Value 0 sets no limit. + maxParallelUpdateGC = 1000 +) + +// DB is the local store implementation and holds +// database related objects. +type DB struct { + shed *shed.DB + + // schema name of loaded data + schemaName shed.StringField + // field that stores number of intems in gc index + storedGCSize shed.Uint64Field + + // retrieval indexes + retrievalDataIndex shed.Index + retrievalAccessIndex shed.Index + // push syncing index + pushIndex shed.Index + // push syncing subscriptions triggers + pushTriggers []chan struct{} + pushTriggersMu sync.RWMutex + + // pull syncing index + pullIndex shed.Index + // pull syncing subscriptions triggers per bin + pullTriggers map[uint8][]chan struct{} + pullTriggersMu sync.RWMutex + + // garbage collection index + gcIndex shed.Index + // index that stores hashes that are not + // counted in and saved to storedGCSize + gcUncountedHashesIndex shed.Index + + // number of elements in garbage collection index + // it must be always read by getGCSize and + // set with incGCSize which are locking gcSizeMu + gcSize int64 + gcSizeMu sync.RWMutex + // garbage collection is triggered when gcSize exceeds + // the capacity value + capacity int64 + + // triggers garbage collection event loop + collectGarbageTrigger chan struct{} + // triggers write gc size event loop + writeGCSizeTrigger chan struct{} + + // a buffered channel acting as a semaphore + // to limit the maximal number of goroutines + // created by Getters to call updateGC function + updateGCSem chan struct{} + // a wait group to ensure all updateGC goroutines + // are done before closing the database + updateGCWG sync.WaitGroup + + baseKey []byte + + addressLocks sync.Map + + // this channel is closed when close function is called + // to terminate other goroutines + close chan struct{} +} + +// Options struct holds optional parameters for configuring DB. +type Options struct { + // MockStore is a mock node store that is used to store + // chunk data in a central store. It can be used to reduce + // total storage space requirements in testing large number + // of swarm nodes with chunk data deduplication provided by + // the mock global store. + MockStore *mock.NodeStore + // Capacity is a limit that triggers garbage collection when + // number of items in gcIndex equals or exceeds it. + Capacity int64 + // MetricsPrefix defines a prefix for metrics names. + MetricsPrefix string +} + +// New returns a new DB. All fields and indexes are initialized +// and possible conflicts with schema from existing database is checked. +// One goroutine for writing batches is created. +func New(path string, baseKey []byte, o *Options) (db *DB, err error) { + if o == nil { + o = new(Options) + } + db = &DB{ + capacity: o.Capacity, + baseKey: baseKey, + // channels collectGarbageTrigger and writeGCSizeTrigger + // need to be buffered with the size of 1 + // to signal another event if it + // is triggered during already running function + collectGarbageTrigger: make(chan struct{}, 1), + writeGCSizeTrigger: make(chan struct{}, 1), + close: make(chan struct{}), + } + if db.capacity <= 0 { + db.capacity = defaultCapacity + } + if maxParallelUpdateGC > 0 { + db.updateGCSem = make(chan struct{}, maxParallelUpdateGC) + } + + db.shed, err = shed.NewDB(path, o.MetricsPrefix) + if err != nil { + return nil, err + } + // Identify current storage schema by arbitrary name. + db.schemaName, err = db.shed.NewStringField("schema-name") + if err != nil { + return nil, err + } + // Persist gc size. + db.storedGCSize, err = db.shed.NewUint64Field("gc-size") + if err != nil { + return nil, err + } + // Functions for retrieval data index. + var ( + encodeValueFunc func(fields shed.Item) (value []byte, err error) + decodeValueFunc func(keyItem shed.Item, value []byte) (e shed.Item, err error) + ) + if o.MockStore != nil { + encodeValueFunc = func(fields shed.Item) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + err = o.MockStore.Put(fields.Address, fields.Data) + if err != nil { + return nil, err + } + return b, nil + } + decodeValueFunc = func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data, err = o.MockStore.Get(keyItem.Address) + return e, err + } + } else { + encodeValueFunc = func(fields shed.Item) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.StoreTimestamp)) + value = append(b, fields.Data...) + return value, nil + } + decodeValueFunc = func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + e.StoreTimestamp = int64(binary.BigEndian.Uint64(value[:8])) + e.Data = value[8:] + return e, nil + } + } + // Index storing actual chunk address, data and store timestamp. + db.retrievalDataIndex, err = db.shed.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key + return e, nil + }, + EncodeValue: encodeValueFunc, + DecodeValue: decodeValueFunc, + }) + if err != nil { + return nil, err + } + // Index storing access timestamp for a particular address. + // It is needed in order to update gc index keys for iteration order. + db.retrievalAccessIndex, err = db.shed.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(fields.AccessTimestamp)) + return b, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) + return e, nil + }, + }) + if err != nil { + return nil, err + } + // pull index allows history and live syncing per po bin + db.pullIndex, err = db.shed.NewIndex("PO|StoredTimestamp|Hash->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + key = make([]byte, 41) + key[0] = db.po(fields.Address) + binary.BigEndian.PutUint64(key[1:9], uint64(fields.StoreTimestamp)) + copy(key[9:], fields.Address[:]) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key[9:] + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[1:9])) + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + // create a pull syncing triggers used by SubscribePull function + db.pullTriggers = make(map[uint8][]chan struct{}) + // push index contains as yet unsynced chunks + db.pushIndex, err = db.shed.NewIndex("StoredTimestamp|Hash->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + key = make([]byte, 40) + binary.BigEndian.PutUint64(key[:8], uint64(fields.StoreTimestamp)) + copy(key[8:], fields.Address[:]) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key[8:] + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + // create a push syncing triggers used by SubscribePush function + db.pushTriggers = make([]chan struct{}, 0) + // gc index for removable chunk ordered by ascending last access time + db.gcIndex, err = db.shed.NewIndex("AccessTimestamp|StoredTimestamp|Hash->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + b := make([]byte, 16, 16+len(fields.Address)) + binary.BigEndian.PutUint64(b[:8], uint64(fields.AccessTimestamp)) + binary.BigEndian.PutUint64(b[8:16], uint64(fields.StoreTimestamp)) + key = append(b, fields.Address...) + return key, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.AccessTimestamp = int64(binary.BigEndian.Uint64(key[:8])) + e.StoreTimestamp = int64(binary.BigEndian.Uint64(key[8:16])) + e.Address = key[16:] + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + // gc uncounted hashes index keeps hashes that are in gc index + // but not counted in and saved to storedGCSize + db.gcUncountedHashesIndex, err = db.shed.NewIndex("Hash->nil", shed.IndexFuncs{ + EncodeKey: func(fields shed.Item) (key []byte, err error) { + return fields.Address, nil + }, + DecodeKey: func(key []byte) (e shed.Item, err error) { + e.Address = key + return e, nil + }, + EncodeValue: func(fields shed.Item) (value []byte, err error) { + return nil, nil + }, + DecodeValue: func(keyItem shed.Item, value []byte) (e shed.Item, err error) { + return e, nil + }, + }) + if err != nil { + return nil, err + } + + // count number of elements in garbage collection index + gcSize, err := db.storedGCSize.Get() + if err != nil { + return nil, err + } + // get number of uncounted hashes + gcUncountedSize, err := db.gcUncountedHashesIndex.Count() + if err != nil { + return nil, err + } + gcSize += uint64(gcUncountedSize) + // remove uncounted hashes from the index and + // save the total gcSize after uncounted hashes are removed + err = db.writeGCSize(int64(gcSize)) + if err != nil { + return nil, err + } + db.incGCSize(int64(gcSize)) + + // start worker to write gc size + go db.writeGCSizeWorker() + // start garbage collection worker + go db.collectGarbageWorker() + return db, nil +} + +// Close closes the underlying database. +func (db *DB) Close() (err error) { + close(db.close) + db.updateGCWG.Wait() + if err := db.writeGCSize(db.getGCSize()); err != nil { + log.Error("localstore: write gc size", "err", err) + } + return db.shed.Close() +} + +// po computes the proximity order between the address +// and database base key. +func (db *DB) po(addr storage.Address) (bin uint8) { + return uint8(storage.Proximity(db.baseKey, addr)) +} + +var ( + // Maximal time for lockAddr to wait until it + // returns error. + addressLockTimeout = 3 * time.Second + // duration between two lock checks in lockAddr. + addressLockCheckDelay = 30 * time.Microsecond +) + +// lockAddr sets the lock on a particular address +// using addressLocks sync.Map and returns unlock function. +// If the address is locked this function will check it +// in a for loop for addressLockTimeout time, after which +// it will return ErrAddressLockTimeout error. +func (db *DB) lockAddr(addr storage.Address) (unlock func(), err error) { + start := time.Now() + lockKey := hex.EncodeToString(addr) + for { + _, loaded := db.addressLocks.LoadOrStore(lockKey, struct{}{}) + if !loaded { + break + } + time.Sleep(addressLockCheckDelay) + if time.Since(start) > addressLockTimeout { + return nil, ErrAddressLockTimeout + } + } + return func() { db.addressLocks.Delete(lockKey) }, nil +} + +// chunkToItem creates new Item with data provided by the Chunk. +func chunkToItem(ch storage.Chunk) shed.Item { + return shed.Item{ + Address: ch.Address(), + Data: ch.Data(), + } +} + +// addressToItem creates new Item with a provided address. +func addressToItem(addr storage.Address) shed.Item { + return shed.Item{ + Address: addr, + } +} + +// now is a helper function that returns a current unix timestamp +// in UTC timezone. +// It is set in the init function for usage in production, and +// optionally overridden in tests for data validation. +var now func() int64 + +func init() { + // set the now function + now = func() (t int64) { + return time.Now().UTC().UnixNano() + } +} diff --git a/swarm/storage/localstore/localstore_test.go b/swarm/storage/localstore/localstore_test.go new file mode 100644 index 0000000000..c7309d3cd8 --- /dev/null +++ b/swarm/storage/localstore/localstore_test.go @@ -0,0 +1,520 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "fmt" + "io/ioutil" + "math/rand" + "os" + "sort" + "strconv" + "sync" + "testing" + "time" + + ch "github.com/ethereum/go-ethereum/swarm/chunk" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// TestDB validates if the chunk can be uploaded and +// correctly retrieved. +func TestDB(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + got, err := db.NewGetter(ModeGetRequest).Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(got.Address(), chunk.Address()) { + t.Errorf("got address %x, want %x", got.Address(), chunk.Address()) + } + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Errorf("got data %x, want %x", got.Data(), chunk.Data()) + } +} + +// TestDB_updateGCSem tests maxParallelUpdateGC limit. +// This test temporary sets the limit to a low number, +// makes updateGC function execution time longer by +// setting a custom testHookUpdateGC function with a sleep +// and a count current and maximal number of goroutines. +func TestDB_updateGCSem(t *testing.T) { + updateGCSleep := time.Second + var count int + var max int + var mu sync.Mutex + defer setTestHookUpdateGC(func() { + mu.Lock() + // add to the count of current goroutines + count++ + if count > max { + // set maximal detected numbers of goroutines + max = count + } + mu.Unlock() + + // wait for some time to ensure multiple parallel goroutines + time.Sleep(updateGCSleep) + + mu.Lock() + count-- + mu.Unlock() + })() + + defer func(m int) { maxParallelUpdateGC = m }(maxParallelUpdateGC) + maxParallelUpdateGC = 3 + + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + getter := db.NewGetter(ModeGetRequest) + + // get more chunks then maxParallelUpdateGC + // in time shorter then updateGCSleep + for i := 0; i < 5; i++ { + _, err = getter.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + } + + if max != maxParallelUpdateGC { + t.Errorf("got max %v, want %v", max, maxParallelUpdateGC) + } +} + +// BenchmarkNew measures the time that New function +// needs to initialize and count the number of key/value +// pairs in GC index. +// This benchmark generates a number of chunks, uploads them, +// sets them to synced state for them to enter the GC index, +// and measures the execution time of New function by creating +// new databases with the same data directory. +// +// This benchmark takes significant amount of time. +// +// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) show +// that New function executes around 1s for database with 1M chunks. +// +// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkNew -v -timeout 20m +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore +// BenchmarkNew/1000-8 200 11672414 ns/op 9570960 B/op 10008 allocs/op +// BenchmarkNew/10000-8 100 14890609 ns/op 10490118 B/op 7759 allocs/op +// BenchmarkNew/100000-8 20 58334080 ns/op 17763157 B/op 22978 allocs/op +// BenchmarkNew/1000000-8 2 748595153 ns/op 45297404 B/op 253242 allocs/op +// PASS +func BenchmarkNew(b *testing.B) { + if testing.Short() { + b.Skip("skipping benchmark in short mode") + } + for _, count := range []int{ + 1000, + 10000, + 100000, + 1000000, + } { + b.Run(strconv.Itoa(count), func(b *testing.B) { + dir, err := ioutil.TempDir("", "localstore-new-benchmark") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(dir) + baseKey := make([]byte, 32) + if _, err := rand.Read(baseKey); err != nil { + b.Fatal(err) + } + db, err := New(dir, baseKey, nil) + if err != nil { + b.Fatal(err) + } + uploader := db.NewPutter(ModePutUpload) + syncer := db.NewSetter(ModeSetSync) + for i := 0; i < count; i++ { + chunk := generateFakeRandomChunk() + err := uploader.Put(chunk) + if err != nil { + b.Fatal(err) + } + err = syncer.Set(chunk.Address()) + if err != nil { + b.Fatal(err) + } + } + err = db.Close() + if err != nil { + b.Fatal(err) + } + b.ResetTimer() + + for n := 0; n < b.N; n++ { + b.StartTimer() + db, err := New(dir, baseKey, nil) + b.StopTimer() + + if err != nil { + b.Fatal(err) + } + err = db.Close() + if err != nil { + b.Fatal(err) + } + } + }) + } +} + +// newTestDB is a helper function that constructs a +// temporary database and returns a cleanup function that must +// be called to remove the data. +func newTestDB(t testing.TB, o *Options) (db *DB, cleanupFunc func()) { + t.Helper() + + dir, err := ioutil.TempDir("", "localstore-test") + if err != nil { + t.Fatal(err) + } + cleanupFunc = func() { os.RemoveAll(dir) } + baseKey := make([]byte, 32) + if _, err := rand.Read(baseKey); err != nil { + t.Fatal(err) + } + db, err = New(dir, baseKey, o) + if err != nil { + cleanupFunc() + t.Fatal(err) + } + cleanupFunc = func() { + err := db.Close() + if err != nil { + t.Error(err) + } + os.RemoveAll(dir) + } + return db, cleanupFunc +} + +// generateRandomChunk generates a valid Chunk with +// data size of default chunk size. +func generateRandomChunk() storage.Chunk { + return storage.GenerateRandomChunk(ch.DefaultSize) +} + +func init() { + // needed for generateFakeRandomChunk + rand.Seed(time.Now().UnixNano()) +} + +// generateFakeRandomChunk generates a Chunk that is not +// valid, but it contains a random key and a random value. +// This function is faster then storage.GenerateRandomChunk +// which generates a valid chunk. +// Some tests in this package do not need valid chunks, just +// random data, and their execution time can be decreased +// using this function. +func generateFakeRandomChunk() storage.Chunk { + data := make([]byte, ch.DefaultSize) + rand.Read(data) + key := make([]byte, 32) + rand.Read(key) + return storage.NewChunk(key, data) +} + +// TestGenerateFakeRandomChunk validates that +// generateFakeRandomChunk returns random data by comparing +// two generated chunks. +func TestGenerateFakeRandomChunk(t *testing.T) { + c1 := generateFakeRandomChunk() + c2 := generateFakeRandomChunk() + addrLen := len(c1.Address()) + if addrLen != 32 { + t.Errorf("first chunk address length %v, want %v", addrLen, 32) + } + dataLen := len(c1.Data()) + if dataLen != ch.DefaultSize { + t.Errorf("first chunk data length %v, want %v", dataLen, ch.DefaultSize) + } + addrLen = len(c2.Address()) + if addrLen != 32 { + t.Errorf("second chunk address length %v, want %v", addrLen, 32) + } + dataLen = len(c2.Data()) + if dataLen != ch.DefaultSize { + t.Errorf("second chunk data length %v, want %v", dataLen, ch.DefaultSize) + } + if bytes.Equal(c1.Address(), c2.Address()) { + t.Error("fake chunks addresses do not differ") + } + if bytes.Equal(c1.Data(), c2.Data()) { + t.Error("fake chunks data bytes do not differ") + } +} + +// newRetrieveIndexesTest returns a test function that validates if the right +// chunk values are in the retrieval indexes. +func newRetrieveIndexesTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0) + + // access index should not be set + wantErr := leveldb.ErrNotFound + item, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + } +} + +// newRetrieveIndexesTestWithAccess returns a test function that validates if the right +// chunk values are in the retrieval indexes when access time must be stored. +func newRetrieveIndexesTestWithAccess(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), chunk.Data(), storeTimestamp, 0) + + if accessTimestamp > 0 { + item, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address())) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), nil, 0, accessTimestamp) + } + } +} + +// newPullIndexTest returns a test function that validates if the right +// chunk values are in the pull index. +func newPullIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.pullIndex.Get(shed.Item{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + }) + if err != wantError { + t.Errorf("got error %v, want %v", err, wantError) + } + if err == nil { + validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0) + } + } +} + +// newPushIndexTest returns a test function that validates if the right +// chunk values are in the push index. +func newPushIndexTest(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.pushIndex.Get(shed.Item{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + }) + if err != wantError { + t.Errorf("got error %v, want %v", err, wantError) + } + if err == nil { + validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0) + } + } +} + +// newGCIndexTest returns a test function that validates if the right +// chunk values are in the push index. +func newGCIndexTest(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { + return func(t *testing.T) { + item, err := db.gcIndex.Get(shed.Item{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + AccessTimestamp: accessTimestamp, + }) + if err != nil { + t.Fatal(err) + } + validateItem(t, item, chunk.Address(), nil, storeTimestamp, accessTimestamp) + } +} + +// newItemsCountTest returns a test function that validates if +// an index contains expected number of key/value pairs. +func newItemsCountTest(i shed.Index, want int) func(t *testing.T) { + return func(t *testing.T) { + var c int + err := i.Iterate(func(item shed.Item) (stop bool, err error) { + c++ + return + }, nil) + if err != nil { + t.Fatal(err) + } + if c != want { + t.Errorf("got %v items in index, want %v", c, want) + } + } +} + +// newIndexGCSizeTest retruns a test function that validates if DB.gcSize +// value is the same as the number of items in DB.gcIndex. +func newIndexGCSizeTest(db *DB) func(t *testing.T) { + return func(t *testing.T) { + var want int64 + err := db.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) { + want++ + return + }, nil) + if err != nil { + t.Fatal(err) + } + got := db.getGCSize() + if got != want { + t.Errorf("got gc size %v, want %v", got, want) + } + } +} + +// testIndexChunk embeds storageChunk with additional data that is stored +// in database. It is used for index values validations. +type testIndexChunk struct { + storage.Chunk + storeTimestamp int64 +} + +// testItemsOrder tests the order of chunks in the index. If sortFunc is not nil, +// chunks will be sorted with it before validation. +func testItemsOrder(t *testing.T, i shed.Index, chunks []testIndexChunk, sortFunc func(i, j int) (less bool)) { + newItemsCountTest(i, len(chunks))(t) + + if sortFunc != nil { + sort.Slice(chunks, sortFunc) + } + + var cursor int + err := i.Iterate(func(item shed.Item) (stop bool, err error) { + want := chunks[cursor].Address() + got := item.Address + if !bytes.Equal(got, want) { + return true, fmt.Errorf("got address %x at position %v, want %x", got, cursor, want) + } + cursor++ + return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } +} + +// validateItem is a helper function that checks Item values. +func validateItem(t *testing.T, item shed.Item, address, data []byte, storeTimestamp, accessTimestamp int64) { + t.Helper() + + if !bytes.Equal(item.Address, address) { + t.Errorf("got item address %x, want %x", item.Address, address) + } + if !bytes.Equal(item.Data, data) { + t.Errorf("got item data %x, want %x", item.Data, data) + } + if item.StoreTimestamp != storeTimestamp { + t.Errorf("got item store timestamp %v, want %v", item.StoreTimestamp, storeTimestamp) + } + if item.AccessTimestamp != accessTimestamp { + t.Errorf("got item access timestamp %v, want %v", item.AccessTimestamp, accessTimestamp) + } +} + +// setNow replaces now function and +// returns a function that will reset it to the +// value before the change. +func setNow(f func() int64) (reset func()) { + current := now + reset = func() { now = current } + now = f + return reset +} + +// TestSetNow tests if setNow function changes now function +// correctly and if its reset function resets the original function. +func TestSetNow(t *testing.T) { + // set the current function after the test finishes + defer func(f func() int64) { now = f }(now) + + // expected value for the unchanged function + var original int64 = 1 + // expected value for the changed function + var changed int64 = 2 + + // define the original (unchanged) functions + now = func() int64 { + return original + } + + // get the time + got := now() + + // test if got variable is set correctly + if got != original { + t.Errorf("got now value %v, want %v", got, original) + } + + // set the new function + reset := setNow(func() int64 { + return changed + }) + + // get the time + got = now() + + // test if got variable is set correctly to changed value + if got != changed { + t.Errorf("got hook value %v, want %v", got, changed) + } + + // set the function to the original one + reset() + + // get the time + got = now() + + // test if got variable is set correctly to original value + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } +} diff --git a/swarm/storage/localstore/mode_get.go b/swarm/storage/localstore/mode_get.go new file mode 100644 index 0000000000..3a69f6e9d4 --- /dev/null +++ b/swarm/storage/localstore/mode_get.go @@ -0,0 +1,154 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// ModeGet enumerates different Getter modes. +type ModeGet int + +// Getter modes. +const ( + // ModeGetRequest: when accessed for retrieval + ModeGetRequest ModeGet = iota + // ModeGetSync: when accessed for syncing or proof of custody request + ModeGetSync +) + +// Getter provides Get method to retrieve Chunks +// from database. +type Getter struct { + db *DB + mode ModeGet +} + +// NewGetter returns a new Getter on database +// with a specific Mode. +func (db *DB) NewGetter(mode ModeGet) *Getter { + return &Getter{ + mode: mode, + db: db, + } +} + +// Get returns a chunk from the database. If the chunk is +// not found storage.ErrChunkNotFound will be returned. +// All required indexes will be updated required by the +// Getter Mode. +func (g *Getter) Get(addr storage.Address) (chunk storage.Chunk, err error) { + out, err := g.db.get(g.mode, addr) + if err != nil { + if err == leveldb.ErrNotFound { + return nil, storage.ErrChunkNotFound + } + return nil, err + } + return storage.NewChunk(out.Address, out.Data), nil +} + +// get returns Item from the retrieval index +// and updates other indexes. +func (db *DB) get(mode ModeGet, addr storage.Address) (out shed.Item, err error) { + item := addressToItem(addr) + + out, err = db.retrievalDataIndex.Get(item) + if err != nil { + return out, err + } + switch mode { + // update the access timestamp and gc index + case ModeGetRequest: + if db.updateGCSem != nil { + // wait before creating new goroutines + // if updateGCSem buffer id full + db.updateGCSem <- struct{}{} + } + db.updateGCWG.Add(1) + go func() { + defer db.updateGCWG.Done() + if db.updateGCSem != nil { + // free a spot in updateGCSem buffer + // for a new goroutine + defer func() { <-db.updateGCSem }() + } + err := db.updateGC(out) + if err != nil { + log.Error("localstore update gc", "err", err) + } + // if gc update hook is defined, call it + if testHookUpdateGC != nil { + testHookUpdateGC() + } + }() + + // no updates to indexes + case ModeGetSync: + default: + return out, ErrInvalidMode + } + return out, nil +} + +// updateGC updates garbage collection index for +// a single item. Provided item is expected to have +// only Address and Data fields with non zero values, +// which is ensured by the get function. +func (db *DB) updateGC(item shed.Item) (err error) { + unlock, err := db.lockAddr(item.Address) + if err != nil { + return err + } + defer unlock() + + batch := new(leveldb.Batch) + + // update accessTimeStamp in retrieve, gc + + i, err := db.retrievalAccessIndex.Get(item) + switch err { + case nil: + item.AccessTimestamp = i.AccessTimestamp + case leveldb.ErrNotFound: + // no chunk accesses + default: + return err + } + if item.AccessTimestamp == 0 { + // chunk is not yet synced + // do not add it to the gc index + return nil + } + // delete current entry from the gc index + db.gcIndex.DeleteInBatch(batch, item) + // update access timestamp + item.AccessTimestamp = now() + // update retrieve access index + db.retrievalAccessIndex.PutInBatch(batch, item) + // add new entry to gc index + db.gcIndex.PutInBatch(batch, item) + + return db.shed.WriteBatch(batch) +} + +// testHookUpdateGC is a hook that can provide +// information when a garbage collection index is updated. +var testHookUpdateGC func() diff --git a/swarm/storage/localstore/mode_get_test.go b/swarm/storage/localstore/mode_get_test.go new file mode 100644 index 0000000000..6615a3b889 --- /dev/null +++ b/swarm/storage/localstore/mode_get_test.go @@ -0,0 +1,237 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "testing" + "time" +) + +// TestModeGetRequest validates ModeGetRequest index values on the provided DB. +func TestModeGetRequest(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploadTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return uploadTimestamp + })() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + requester := db.NewGetter(ModeGetRequest) + + // set update gc test hook to signal when + // update gc goroutine is done by sending to + // testHookUpdateGCChan channel, which is + // used to wait for garbage colletion index + // changes + testHookUpdateGCChan := make(chan struct{}) + defer setTestHookUpdateGC(func() { + testHookUpdateGCChan <- struct{}{} + })() + + t.Run("get unsynced", func(t *testing.T) { + got, err := requester.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + + if !bytes.Equal(got.Address(), chunk.Address()) { + t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address()) + } + + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data()) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, 0)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 0)) + + t.Run("gc size", newIndexGCSizeTest(db)) + }) + + // set chunk to synced state + err = db.NewSetter(ModeSetSync).Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + t.Run("first get", func(t *testing.T) { + got, err := requester.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + + if !bytes.Equal(got.Address(), chunk.Address()) { + t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address()) + } + + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data()) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, uploadTimestamp)) + + t.Run("gc index", newGCIndexTest(db, chunk, uploadTimestamp, uploadTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) + }) + + t.Run("second get", func(t *testing.T) { + accessTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return accessTimestamp + })() + + got, err := requester.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + + if !bytes.Equal(got.Address(), chunk.Address()) { + t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address()) + } + + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data()) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, accessTimestamp)) + + t.Run("gc index", newGCIndexTest(db, chunk, uploadTimestamp, accessTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) + }) +} + +// TestModeGetSync validates ModeGetSync index values on the provided DB. +func TestModeGetSync(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploadTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return uploadTimestamp + })() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + got, err := db.NewGetter(ModeGetSync).Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(got.Address(), chunk.Address()) { + t.Errorf("got chunk address %x, want %x", got.Address(), chunk.Address()) + } + + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Errorf("got chunk data %x, want %x", got.Data(), chunk.Data()) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, uploadTimestamp, 0)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 0)) + + t.Run("gc size", newIndexGCSizeTest(db)) +} + +// setTestHookUpdateGC sets testHookUpdateGC and +// returns a function that will reset it to the +// value before the change. +func setTestHookUpdateGC(h func()) (reset func()) { + current := testHookUpdateGC + reset = func() { testHookUpdateGC = current } + testHookUpdateGC = h + return reset +} + +// TestSetTestHookUpdateGC tests if setTestHookUpdateGC changes +// testHookUpdateGC function correctly and if its reset function +// resets the original function. +func TestSetTestHookUpdateGC(t *testing.T) { + // Set the current function after the test finishes. + defer func(h func()) { testHookUpdateGC = h }(testHookUpdateGC) + + // expected value for the unchanged function + original := 1 + // expected value for the changed function + changed := 2 + + // this variable will be set with two different functions + var got int + + // define the original (unchanged) functions + testHookUpdateGC = func() { + got = original + } + + // set got variable + testHookUpdateGC() + + // test if got variable is set correctly + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } + + // set the new function + reset := setTestHookUpdateGC(func() { + got = changed + }) + + // set got variable + testHookUpdateGC() + + // test if got variable is set correctly to changed value + if got != changed { + t.Errorf("got hook value %v, want %v", got, changed) + } + + // set the function to the original one + reset() + + // set got variable + testHookUpdateGC() + + // test if got variable is set correctly to original value + if got != original { + t.Errorf("got hook value %v, want %v", got, original) + } +} diff --git a/swarm/storage/localstore/mode_put.go b/swarm/storage/localstore/mode_put.go new file mode 100644 index 0000000000..1a5a3d1b10 --- /dev/null +++ b/swarm/storage/localstore/mode_put.go @@ -0,0 +1,160 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// ModePut enumerates different Putter modes. +type ModePut int + +// Putter modes. +const ( + // ModePutRequest: when a chunk is received as a result of retrieve request and delivery + ModePutRequest ModePut = iota + // ModePutSync: when a chunk is received via syncing + ModePutSync + // ModePutUpload: when a chunk is created by local upload + ModePutUpload +) + +// Putter provides Put method to store Chunks +// to database. +type Putter struct { + db *DB + mode ModePut +} + +// NewPutter returns a new Putter on database +// with a specific Mode. +func (db *DB) NewPutter(mode ModePut) *Putter { + return &Putter{ + mode: mode, + db: db, + } +} + +// Put stores the Chunk to database and depending +// on the Putter mode, it updates required indexes. +func (p *Putter) Put(ch storage.Chunk) (err error) { + return p.db.put(p.mode, chunkToItem(ch)) +} + +// put stores Item to database and updates other +// indexes. It acquires lockAddr to protect two calls +// of this function for the same address in parallel. +// Item fields Address and Data must not be +// with their nil values. +func (db *DB) put(mode ModePut, item shed.Item) (err error) { + // protect parallel updates + unlock, err := db.lockAddr(item.Address) + if err != nil { + return err + } + defer unlock() + + batch := new(leveldb.Batch) + + // variables that provide information for operations + // to be done after write batch function successfully executes + var gcSizeChange int64 // number to add or subtract from gcSize + var triggerPullFeed bool // signal pull feed subscriptions to iterate + var triggerPushFeed bool // signal push feed subscriptions to iterate + + switch mode { + case ModePutRequest: + // put to indexes: retrieve, gc; it does not enter the syncpool + + // check if the chunk already is in the database + // as gc index is updated + i, err := db.retrievalAccessIndex.Get(item) + switch err { + case nil: + item.AccessTimestamp = i.AccessTimestamp + case leveldb.ErrNotFound: + // no chunk accesses + default: + return err + } + i, err = db.retrievalDataIndex.Get(item) + switch err { + case nil: + item.StoreTimestamp = i.StoreTimestamp + case leveldb.ErrNotFound: + // no chunk accesses + default: + return err + } + if item.AccessTimestamp != 0 { + // delete current entry from the gc index + db.gcIndex.DeleteInBatch(batch, item) + gcSizeChange-- + } + if item.StoreTimestamp == 0 { + item.StoreTimestamp = now() + } + // update access timestamp + item.AccessTimestamp = now() + // update retrieve access index + db.retrievalAccessIndex.PutInBatch(batch, item) + // add new entry to gc index + db.gcIndex.PutInBatch(batch, item) + db.gcUncountedHashesIndex.PutInBatch(batch, item) + gcSizeChange++ + + db.retrievalDataIndex.PutInBatch(batch, item) + + case ModePutUpload: + // put to indexes: retrieve, push, pull + + item.StoreTimestamp = now() + db.retrievalDataIndex.PutInBatch(batch, item) + db.pullIndex.PutInBatch(batch, item) + triggerPullFeed = true + db.pushIndex.PutInBatch(batch, item) + triggerPushFeed = true + + case ModePutSync: + // put to indexes: retrieve, pull + + item.StoreTimestamp = now() + db.retrievalDataIndex.PutInBatch(batch, item) + db.pullIndex.PutInBatch(batch, item) + triggerPullFeed = true + + default: + return ErrInvalidMode + } + + err = db.shed.WriteBatch(batch) + if err != nil { + return err + } + if gcSizeChange != 0 { + db.incGCSize(gcSizeChange) + } + if triggerPullFeed { + db.triggerPullSubscriptions(db.po(item.Address)) + } + if triggerPushFeed { + db.triggerPushSubscriptions() + } + return nil +} diff --git a/swarm/storage/localstore/mode_put_test.go b/swarm/storage/localstore/mode_put_test.go new file mode 100644 index 0000000000..ffe6a4cb4e --- /dev/null +++ b/swarm/storage/localstore/mode_put_test.go @@ -0,0 +1,300 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "fmt" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TestModePutRequest validates ModePutRequest index values on the provided DB. +func TestModePutRequest(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + putter := db.NewPutter(ModePutRequest) + + chunk := generateRandomChunk() + + // keep the record when the chunk is stored + var storeTimestamp int64 + + t.Run("first put", func(t *testing.T) { + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + storeTimestamp = wantTimestamp + + err := putter.Put(chunk) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, wantTimestamp, wantTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) + }) + + t.Run("second put", func(t *testing.T) { + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + err := putter.Put(chunk) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, storeTimestamp, wantTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) + }) +} + +// TestModePutSync validates ModePutSync index values on the provided DB. +func TestModePutSync(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutSync).Put(chunk) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTest(db, chunk, wantTimestamp, 0)) + + t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil)) +} + +// TestModePutUpload validates ModePutUpload index values on the provided DB. +func TestModePutUpload(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTest(db, chunk, wantTimestamp, 0)) + + t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil)) + + t.Run("push index", newPushIndexTest(db, chunk, wantTimestamp, nil)) +} + +// TestModePutUpload_parallel uploads chunks in parallel +// and validates if all chunks can be retrieved with correct data. +func TestModePutUpload_parallel(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunkCount := 1000 + workerCount := 100 + + chunkChan := make(chan storage.Chunk) + errChan := make(chan error) + doneChan := make(chan struct{}) + defer close(doneChan) + + // start uploader workers + for i := 0; i < workerCount; i++ { + go func(i int) { + uploader := db.NewPutter(ModePutUpload) + for { + select { + case chunk, ok := <-chunkChan: + if !ok { + return + } + err := uploader.Put(chunk) + select { + case errChan <- err: + case <-doneChan: + } + case <-doneChan: + return + } + } + }(i) + } + + chunks := make([]storage.Chunk, 0) + var chunksMu sync.Mutex + + // send chunks to workers + go func() { + for i := 0; i < chunkCount; i++ { + chunk := generateRandomChunk() + select { + case chunkChan <- chunk: + case <-doneChan: + return + } + chunksMu.Lock() + chunks = append(chunks, chunk) + chunksMu.Unlock() + } + + close(chunkChan) + }() + + // validate every error from workers + for i := 0; i < chunkCount; i++ { + err := <-errChan + if err != nil { + t.Fatal(err) + } + } + + // get every chunk and validate its data + getter := db.NewGetter(ModeGetRequest) + + chunksMu.Lock() + defer chunksMu.Unlock() + for _, chunk := range chunks { + got, err := getter.Get(chunk.Address()) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got.Data(), chunk.Data()) { + t.Fatalf("got chunk %s data %x, want %x", chunk.Address().Hex(), got.Data(), chunk.Data()) + } + } +} + +// BenchmarkPutUpload runs a series of benchmarks that upload +// a specific number of chunks in parallel. +// +// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) +// +// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkPutUpload -v +// +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore +// BenchmarkPutUpload/count_100_parallel_1-8 300 5107704 ns/op 2081461 B/op 2374 allocs/op +// BenchmarkPutUpload/count_100_parallel_2-8 300 5411742 ns/op 2081608 B/op 2364 allocs/op +// BenchmarkPutUpload/count_100_parallel_4-8 500 3704964 ns/op 2081696 B/op 2324 allocs/op +// BenchmarkPutUpload/count_100_parallel_8-8 500 2932663 ns/op 2082594 B/op 2295 allocs/op +// BenchmarkPutUpload/count_100_parallel_16-8 500 3117157 ns/op 2085438 B/op 2282 allocs/op +// BenchmarkPutUpload/count_100_parallel_32-8 500 3449122 ns/op 2089721 B/op 2286 allocs/op +// BenchmarkPutUpload/count_1000_parallel_1-8 20 79784470 ns/op 25211240 B/op 23225 allocs/op +// BenchmarkPutUpload/count_1000_parallel_2-8 20 75422164 ns/op 25210730 B/op 23187 allocs/op +// BenchmarkPutUpload/count_1000_parallel_4-8 20 70698378 ns/op 25206522 B/op 22692 allocs/op +// BenchmarkPutUpload/count_1000_parallel_8-8 20 71285528 ns/op 25213436 B/op 22345 allocs/op +// BenchmarkPutUpload/count_1000_parallel_16-8 20 71301826 ns/op 25205040 B/op 22090 allocs/op +// BenchmarkPutUpload/count_1000_parallel_32-8 30 57713506 ns/op 25219781 B/op 21848 allocs/op +// BenchmarkPutUpload/count_10000_parallel_1-8 2 656719345 ns/op 216792908 B/op 248940 allocs/op +// BenchmarkPutUpload/count_10000_parallel_2-8 2 646301962 ns/op 216730800 B/op 248270 allocs/op +// BenchmarkPutUpload/count_10000_parallel_4-8 2 532784228 ns/op 216667080 B/op 241910 allocs/op +// BenchmarkPutUpload/count_10000_parallel_8-8 3 494290188 ns/op 216297749 B/op 236247 allocs/op +// BenchmarkPutUpload/count_10000_parallel_16-8 3 483485315 ns/op 216060384 B/op 231090 allocs/op +// BenchmarkPutUpload/count_10000_parallel_32-8 3 434461294 ns/op 215371280 B/op 224800 allocs/op +// BenchmarkPutUpload/count_100000_parallel_1-8 1 22767894338 ns/op 2331372088 B/op 4049876 allocs/op +// BenchmarkPutUpload/count_100000_parallel_2-8 1 25347872677 ns/op 2344140160 B/op 4106763 allocs/op +// BenchmarkPutUpload/count_100000_parallel_4-8 1 23580460174 ns/op 2338582576 B/op 4027452 allocs/op +// BenchmarkPutUpload/count_100000_parallel_8-8 1 22197559193 ns/op 2321803496 B/op 3877553 allocs/op +// BenchmarkPutUpload/count_100000_parallel_16-8 1 22527046476 ns/op 2327854800 B/op 3885455 allocs/op +// BenchmarkPutUpload/count_100000_parallel_32-8 1 21332243613 ns/op 2299654568 B/op 3697181 allocs/op +// PASS +func BenchmarkPutUpload(b *testing.B) { + for _, count := range []int{ + 100, + 1000, + 10000, + 100000, + } { + for _, maxParallelUploads := range []int{ + 1, + 2, + 4, + 8, + 16, + 32, + } { + name := fmt.Sprintf("count %v parallel %v", count, maxParallelUploads) + b.Run(name, func(b *testing.B) { + for n := 0; n < b.N; n++ { + benchmarkPutUpload(b, nil, count, maxParallelUploads) + } + }) + } + } +} + +// benchmarkPutUpload runs a benchmark by uploading a specific number +// of chunks with specified max parallel uploads. +func benchmarkPutUpload(b *testing.B, o *Options, count, maxParallelUploads int) { + b.StopTimer() + db, cleanupFunc := newTestDB(b, o) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + chunks := make([]storage.Chunk, count) + for i := 0; i < count; i++ { + chunks[i] = generateFakeRandomChunk() + } + errs := make(chan error) + b.StartTimer() + + go func() { + sem := make(chan struct{}, maxParallelUploads) + for i := 0; i < count; i++ { + sem <- struct{}{} + + go func(i int) { + defer func() { <-sem }() + + errs <- uploader.Put(chunks[i]) + }(i) + } + }() + + for i := 0; i < count; i++ { + err := <-errs + if err != nil { + b.Fatal(err) + } + } +} diff --git a/swarm/storage/localstore/mode_set.go b/swarm/storage/localstore/mode_set.go new file mode 100644 index 0000000000..a522f4447c --- /dev/null +++ b/swarm/storage/localstore/mode_set.go @@ -0,0 +1,205 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/syndtr/goleveldb/leveldb" +) + +// ModeSet enumerates different Setter modes. +type ModeSet int + +// Setter modes. +const ( + // ModeSetAccess: when an update request is received for a chunk or chunk is retrieved for delivery + ModeSetAccess ModeSet = iota + // ModeSetSync: when push sync receipt is received + ModeSetSync + // modeSetRemove: when GC-d + // unexported as no external packages should remove chunks from database + modeSetRemove +) + +// Setter sets the state of a particular +// Chunk in database by changing indexes. +type Setter struct { + db *DB + mode ModeSet +} + +// NewSetter returns a new Setter on database +// with a specific Mode. +func (db *DB) NewSetter(mode ModeSet) *Setter { + return &Setter{ + mode: mode, + db: db, + } +} + +// Set updates database indexes for a specific +// chunk represented by the address. +func (s *Setter) Set(addr storage.Address) (err error) { + return s.db.set(s.mode, addr) +} + +// set updates database indexes for a specific +// chunk represented by the address. +// It acquires lockAddr to protect two calls +// of this function for the same address in parallel. +func (db *DB) set(mode ModeSet, addr storage.Address) (err error) { + // protect parallel updates + unlock, err := db.lockAddr(addr) + if err != nil { + return err + } + defer unlock() + + batch := new(leveldb.Batch) + + // variables that provide information for operations + // to be done after write batch function successfully executes + var gcSizeChange int64 // number to add or subtract from gcSize + var triggerPullFeed bool // signal pull feed subscriptions to iterate + + item := addressToItem(addr) + + switch mode { + case ModeSetAccess: + // add to pull, insert to gc + + // need to get access timestamp here as it is not + // provided by the access function, and it is not + // a property of a chunk provided to Accessor.Put. + + i, err := db.retrievalDataIndex.Get(item) + switch err { + case nil: + item.StoreTimestamp = i.StoreTimestamp + case leveldb.ErrNotFound: + db.pushIndex.DeleteInBatch(batch, item) + item.StoreTimestamp = now() + default: + return err + } + + i, err = db.retrievalAccessIndex.Get(item) + switch err { + case nil: + item.AccessTimestamp = i.AccessTimestamp + db.gcIndex.DeleteInBatch(batch, item) + gcSizeChange-- + case leveldb.ErrNotFound: + // the chunk is not accessed before + default: + return err + } + item.AccessTimestamp = now() + db.retrievalAccessIndex.PutInBatch(batch, item) + db.pullIndex.PutInBatch(batch, item) + triggerPullFeed = true + db.gcIndex.PutInBatch(batch, item) + db.gcUncountedHashesIndex.PutInBatch(batch, item) + gcSizeChange++ + + case ModeSetSync: + // delete from push, insert to gc + + // need to get access timestamp here as it is not + // provided by the access function, and it is not + // a property of a chunk provided to Accessor.Put. + i, err := db.retrievalDataIndex.Get(item) + if err != nil { + if err == leveldb.ErrNotFound { + // chunk is not found, + // no need to update gc index + // just delete from the push index + // if it is there + db.pushIndex.DeleteInBatch(batch, item) + return nil + } + return err + } + item.StoreTimestamp = i.StoreTimestamp + + i, err = db.retrievalAccessIndex.Get(item) + switch err { + case nil: + item.AccessTimestamp = i.AccessTimestamp + db.gcIndex.DeleteInBatch(batch, item) + gcSizeChange-- + case leveldb.ErrNotFound: + // the chunk is not accessed before + default: + return err + } + item.AccessTimestamp = now() + db.retrievalAccessIndex.PutInBatch(batch, item) + db.pushIndex.DeleteInBatch(batch, item) + db.gcIndex.PutInBatch(batch, item) + db.gcUncountedHashesIndex.PutInBatch(batch, item) + gcSizeChange++ + + case modeSetRemove: + // delete from retrieve, pull, gc + + // need to get access timestamp here as it is not + // provided by the access function, and it is not + // a property of a chunk provided to Accessor.Put. + + i, err := db.retrievalAccessIndex.Get(item) + switch err { + case nil: + item.AccessTimestamp = i.AccessTimestamp + case leveldb.ErrNotFound: + default: + return err + } + i, err = db.retrievalDataIndex.Get(item) + if err != nil { + return err + } + item.StoreTimestamp = i.StoreTimestamp + + db.retrievalDataIndex.DeleteInBatch(batch, item) + db.retrievalAccessIndex.DeleteInBatch(batch, item) + db.pullIndex.DeleteInBatch(batch, item) + db.gcIndex.DeleteInBatch(batch, item) + db.gcUncountedHashesIndex.DeleteInBatch(batch, item) + // a check is needed for decrementing gcSize + // as delete is not reporting if the key/value pair + // is deleted or not + if _, err := db.gcIndex.Get(item); err == nil { + gcSizeChange = -1 + } + + default: + return ErrInvalidMode + } + + err = db.shed.WriteBatch(batch) + if err != nil { + return err + } + if gcSizeChange != 0 { + db.incGCSize(gcSizeChange) + } + if triggerPullFeed { + db.triggerPullSubscriptions(db.po(item.Address)) + } + return nil +} diff --git a/swarm/storage/localstore/mode_set_test.go b/swarm/storage/localstore/mode_set_test.go new file mode 100644 index 0000000000..94cd0a3e2c --- /dev/null +++ b/swarm/storage/localstore/mode_set_test.go @@ -0,0 +1,128 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "testing" + "time" + + "github.com/syndtr/goleveldb/leveldb" +) + +// TestModeSetAccess validates ModeSetAccess index values on the provided DB. +func TestModeSetAccess(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunk := generateRandomChunk() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + err := db.NewSetter(ModeSetAccess).Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + t.Run("pull index", newPullIndexTest(db, chunk, wantTimestamp, nil)) + + t.Run("pull index count", newItemsCountTest(db.pullIndex, 1)) + + t.Run("gc index", newGCIndexTest(db, chunk, wantTimestamp, wantTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) +} + +// TestModeSetSync validates ModeSetSync index values on the provided DB. +func TestModeSetSync(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunk := generateRandomChunk() + + wantTimestamp := time.Now().UTC().UnixNano() + defer setNow(func() (t int64) { + return wantTimestamp + })() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + err = db.NewSetter(ModeSetSync).Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", newRetrieveIndexesTestWithAccess(db, chunk, wantTimestamp, wantTimestamp)) + + t.Run("push index", newPushIndexTest(db, chunk, wantTimestamp, leveldb.ErrNotFound)) + + t.Run("gc index", newGCIndexTest(db, chunk, wantTimestamp, wantTimestamp)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 1)) + + t.Run("gc size", newIndexGCSizeTest(db)) +} + +// TestModeSetRemove validates ModeSetRemove index values on the provided DB. +func TestModeSetRemove(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + chunk := generateRandomChunk() + + err := db.NewPutter(ModePutUpload).Put(chunk) + if err != nil { + t.Fatal(err) + } + + err = db.NewSetter(modeSetRemove).Set(chunk.Address()) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", func(t *testing.T) { + wantErr := leveldb.ErrNotFound + _, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + t.Run("retrieve data index count", newItemsCountTest(db.retrievalDataIndex, 0)) + + // access index should not be set + _, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + t.Run("retrieve access index count", newItemsCountTest(db.retrievalAccessIndex, 0)) + }) + + t.Run("pull index", newPullIndexTest(db, chunk, 0, leveldb.ErrNotFound)) + + t.Run("pull index count", newItemsCountTest(db.pullIndex, 0)) + + t.Run("gc index count", newItemsCountTest(db.gcIndex, 0)) + + t.Run("gc size", newIndexGCSizeTest(db)) + +} diff --git a/swarm/storage/localstore/retrieval_index_test.go b/swarm/storage/localstore/retrieval_index_test.go new file mode 100644 index 0000000000..9f5b452c5c --- /dev/null +++ b/swarm/storage/localstore/retrieval_index_test.go @@ -0,0 +1,150 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "strconv" + "testing" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// BenchmarkRetrievalIndexes uploads a number of chunks in order to measure +// total time of updating their retrieval indexes by setting them +// to synced state and requesting them. +// +// This benchmark takes significant amount of time. +// +// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014) show +// that two separated indexes perform better. +// +// # go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkRetrievalIndexes -v +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore +// BenchmarkRetrievalIndexes/1000-8 20 75556686 ns/op 19033493 B/op 84500 allocs/op +// BenchmarkRetrievalIndexes/10000-8 1 1079084922 ns/op 382792064 B/op 1429644 allocs/op +// BenchmarkRetrievalIndexes/100000-8 1 16891305737 ns/op 2629165304 B/op 12465019 allocs/op +// PASS +func BenchmarkRetrievalIndexes(b *testing.B) { + for _, count := range []int{ + 1000, + 10000, + 100000, + } { + b.Run(strconv.Itoa(count)+"-split", func(b *testing.B) { + for n := 0; n < b.N; n++ { + benchmarkRetrievalIndexes(b, nil, count) + } + }) + } +} + +// benchmarkRetrievalIndexes is used in BenchmarkRetrievalIndexes +// to do benchmarks with a specific number of chunks and different +// database options. +func benchmarkRetrievalIndexes(b *testing.B, o *Options, count int) { + b.StopTimer() + db, cleanupFunc := newTestDB(b, o) + defer cleanupFunc() + uploader := db.NewPutter(ModePutUpload) + syncer := db.NewSetter(ModeSetSync) + requester := db.NewGetter(ModeGetRequest) + addrs := make([]storage.Address, count) + for i := 0; i < count; i++ { + chunk := generateFakeRandomChunk() + err := uploader.Put(chunk) + if err != nil { + b.Fatal(err) + } + addrs[i] = chunk.Address() + } + // set update gc test hook to signal when + // update gc goroutine is done by sending to + // testHookUpdateGCChan channel, which is + // used to wait for gc index updates to be + // included in the benchmark time + testHookUpdateGCChan := make(chan struct{}) + defer setTestHookUpdateGC(func() { + testHookUpdateGCChan <- struct{}{} + })() + b.StartTimer() + + for i := 0; i < count; i++ { + err := syncer.Set(addrs[i]) + if err != nil { + b.Fatal(err) + } + + _, err = requester.Get(addrs[i]) + if err != nil { + b.Fatal(err) + } + // wait for update gc goroutine to be done + <-testHookUpdateGCChan + } +} + +// BenchmarkUpload compares uploading speed for different +// retrieval indexes and various number of chunks. +// +// Measurements on MacBook Pro (Retina, 15-inch, Mid 2014). +// +// go test -benchmem -run=none github.com/ethereum/go-ethereum/swarm/storage/localstore -bench BenchmarkUpload -v +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/swarm/storage/localstore +// BenchmarkUpload/1000-8 20 59437463 ns/op 25205193 B/op 23208 allocs/op +// BenchmarkUpload/10000-8 2 580646362 ns/op 216532932 B/op 248090 allocs/op +// BenchmarkUpload/100000-8 1 22373390892 ns/op 2323055312 B/op 3995903 allocs/op +// PASS +func BenchmarkUpload(b *testing.B) { + for _, count := range []int{ + 1000, + 10000, + 100000, + } { + b.Run(strconv.Itoa(count), func(b *testing.B) { + for n := 0; n < b.N; n++ { + benchmarkUpload(b, nil, count) + } + }) + } +} + +// benchmarkUpload is used in BenchmarkUpload +// to do benchmarks with a specific number of chunks and different +// database options. +func benchmarkUpload(b *testing.B, o *Options, count int) { + b.StopTimer() + db, cleanupFunc := newTestDB(b, o) + defer cleanupFunc() + uploader := db.NewPutter(ModePutUpload) + chunks := make([]storage.Chunk, count) + for i := 0; i < count; i++ { + chunk := generateFakeRandomChunk() + chunks[i] = chunk + } + b.StartTimer() + + for i := 0; i < count; i++ { + err := uploader.Put(chunks[i]) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/swarm/storage/localstore/subscription_pull.go b/swarm/storage/localstore/subscription_pull.go new file mode 100644 index 0000000000..a18f0915d9 --- /dev/null +++ b/swarm/storage/localstore/subscription_pull.go @@ -0,0 +1,193 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// SubscribePull returns a channel that provides chunk addresses and stored times from pull syncing index. +// Pull syncing index can be only subscribed to a particular proximity order bin. If since +// is not nil, the iteration will start from the first item stored after that timestamp. If until is not nil, +// only chunks stored up to this timestamp will be send to the channel, and the returned channel will be +// closed. The since-until interval is open on the left and closed on the right (since,until]. Returned stop +// function will terminate current and further iterations without errors, and also close the returned channel. +// Make sure that you check the second returned parameter from the channel to stop iteration when its value +// is false. +func (db *DB) SubscribePull(ctx context.Context, bin uint8, since, until *ChunkDescriptor) (c <-chan ChunkDescriptor, stop func()) { + chunkDescriptors := make(chan ChunkDescriptor) + trigger := make(chan struct{}, 1) + + db.pullTriggersMu.Lock() + if _, ok := db.pullTriggers[bin]; !ok { + db.pullTriggers[bin] = make([]chan struct{}, 0) + } + db.pullTriggers[bin] = append(db.pullTriggers[bin], trigger) + db.pullTriggersMu.Unlock() + + // send signal for the initial iteration + trigger <- struct{}{} + + stopChan := make(chan struct{}) + var stopChanOnce sync.Once + + // used to provide information from the iterator to + // stop subscription when until chunk descriptor is reached + var errStopSubscription = errors.New("stop subscription") + + go func() { + // close the returned ChunkDescriptor channel at the end to + // signal that the subscription is done + defer close(chunkDescriptors) + // sinceItem is the Item from which the next iteration + // should start. The first iteration starts from the first Item. + var sinceItem *shed.Item + if since != nil { + sinceItem = &shed.Item{ + Address: since.Address, + StoreTimestamp: since.StoreTimestamp, + } + } + for { + select { + case <-trigger: + // iterate until: + // - last index Item is reached + // - subscription stop is called + // - context is done + err := db.pullIndex.Iterate(func(item shed.Item) (stop bool, err error) { + select { + case chunkDescriptors <- ChunkDescriptor{ + Address: item.Address, + StoreTimestamp: item.StoreTimestamp, + }: + // until chunk descriptor is sent + // break the iteration + if until != nil && + (item.StoreTimestamp >= until.StoreTimestamp || + bytes.Equal(item.Address, until.Address)) { + return true, errStopSubscription + } + // set next iteration start item + // when its chunk is successfully sent to channel + sinceItem = &item + return false, nil + case <-stopChan: + // gracefully stop the iteration + // on stop + return true, nil + case <-db.close: + // gracefully stop the iteration + // on database close + return true, nil + case <-ctx.Done(): + return true, ctx.Err() + } + }, &shed.IterateOptions{ + StartFrom: sinceItem, + // sinceItem was sent as the last Address in the previous + // iterator call, skip it in this one + SkipStartFromItem: true, + Prefix: []byte{bin}, + }) + if err != nil { + if err == errStopSubscription { + // stop subscription without any errors + // if until is reached + return + } + log.Error("localstore pull subscription iteration", "bin", bin, "since", since, "until", until, "err", err) + return + } + case <-stopChan: + // terminate the subscription + // on stop + return + case <-db.close: + // terminate the subscription + // on database close + return + case <-ctx.Done(): + err := ctx.Err() + if err != nil { + log.Error("localstore pull subscription", "bin", bin, "since", since, "until", until, "err", err) + } + return + } + } + }() + + stop = func() { + stopChanOnce.Do(func() { + close(stopChan) + }) + + db.pullTriggersMu.Lock() + defer db.pullTriggersMu.Unlock() + + for i, t := range db.pullTriggers[bin] { + if t == trigger { + db.pullTriggers[bin] = append(db.pullTriggers[bin][:i], db.pullTriggers[bin][i+1:]...) + break + } + } + } + + return chunkDescriptors, stop +} + +// ChunkDescriptor holds information required for Pull syncing. This struct +// is provided by subscribing to pull index. +type ChunkDescriptor struct { + Address storage.Address + StoreTimestamp int64 +} + +func (c *ChunkDescriptor) String() string { + if c == nil { + return "none" + } + return fmt.Sprintf("%s stored at %v", c.Address.Hex(), c.StoreTimestamp) +} + +// triggerPullSubscriptions is used internally for starting iterations +// on Pull subscriptions for a particular bin. When new item with address +// that is in particular bin for DB's baseKey is added to pull index +// this function should be called. +func (db *DB) triggerPullSubscriptions(bin uint8) { + db.pullTriggersMu.RLock() + triggers, ok := db.pullTriggers[bin] + db.pullTriggersMu.RUnlock() + if !ok { + return + } + + for _, t := range triggers { + select { + case t <- struct{}{}: + default: + } + } +} diff --git a/swarm/storage/localstore/subscription_pull_test.go b/swarm/storage/localstore/subscription_pull_test.go new file mode 100644 index 0000000000..5c99e0dec2 --- /dev/null +++ b/swarm/storage/localstore/subscription_pull_test.go @@ -0,0 +1,478 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TestDB_SubscribePull uploads some chunks before and after +// pull syncing subscription is created and validates if +// all addresses are received in the right order +// for expected proximity order bins. +func TestDB_SubscribePull(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make(map[uint8][]storage.Address) + var addrsMu sync.Mutex + var wantedChunksCount int + + // prepopulate database with some chunks + // before the subscription + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 10) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + ch, stop := db.SubscribePull(ctx, bin, nil, nil) + defer stop() + + // receive and validate addresses from the subscription + go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan) + } + + // upload some chunks just after subscribe + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 5) + + time.Sleep(200 * time.Millisecond) + + // upload some chunks after some short time + // to ensure that subscription will include them + // in a dynamic environment + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 3) + + checkErrChan(ctx, t, errChan, wantedChunksCount) +} + +// TestDB_SubscribePull_multiple uploads chunks before and after +// multiple pull syncing subscriptions are created and +// validates if all addresses are received in the right order +// for expected proximity order bins. +func TestDB_SubscribePull_multiple(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make(map[uint8][]storage.Address) + var addrsMu sync.Mutex + var wantedChunksCount int + + // prepopulate database with some chunks + // before the subscription + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 10) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + subsCount := 10 + + // start a number of subscriptions + // that all of them will write every address error to errChan + for j := 0; j < subsCount; j++ { + for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + ch, stop := db.SubscribePull(ctx, bin, nil, nil) + defer stop() + + // receive and validate addresses from the subscription + go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan) + } + } + + // upload some chunks just after subscribe + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 5) + + time.Sleep(200 * time.Millisecond) + + // upload some chunks after some short time + // to ensure that subscription will include them + // in a dynamic environment + uploadRandomChunksBin(t, db, uploader, addrs, &addrsMu, &wantedChunksCount, 3) + + checkErrChan(ctx, t, errChan, wantedChunksCount*subsCount) +} + +// TestDB_SubscribePull_since uploads chunks before and after +// pull syncing subscriptions are created with a since argument +// and validates if all expected addresses are received in the +// right order for expected proximity order bins. +func TestDB_SubscribePull_since(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make(map[uint8][]storage.Address) + var addrsMu sync.Mutex + var wantedChunksCount int + + lastTimestamp := time.Now().UTC().UnixNano() + var lastTimestampMu sync.RWMutex + defer setNow(func() (t int64) { + lastTimestampMu.Lock() + defer lastTimestampMu.Unlock() + lastTimestamp++ + return lastTimestamp + })() + + uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + last = make(map[uint8]ChunkDescriptor) + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + bin := db.po(chunk.Address()) + + addrsMu.Lock() + if _, ok := addrs[bin]; !ok { + addrs[bin] = make([]storage.Address, 0) + } + if wanted { + addrs[bin] = append(addrs[bin], chunk.Address()) + wantedChunksCount++ + } + addrsMu.Unlock() + + lastTimestampMu.RLock() + storeTimestamp := lastTimestamp + lastTimestampMu.RUnlock() + + last[bin] = ChunkDescriptor{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + } + } + return last + } + + // prepopulate database with some chunks + // before the subscription + last := uploadRandomChunks(30, false) + + uploadRandomChunks(25, true) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + var since *ChunkDescriptor + if c, ok := last[bin]; ok { + since = &c + } + ch, stop := db.SubscribePull(ctx, bin, since, nil) + defer stop() + + // receive and validate addresses from the subscription + go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan) + + } + + // upload some chunks just after subscribe + uploadRandomChunks(15, true) + + checkErrChan(ctx, t, errChan, wantedChunksCount) +} + +// TestDB_SubscribePull_until uploads chunks before and after +// pull syncing subscriptions are created with an until argument +// and validates if all expected addresses are received in the +// right order for expected proximity order bins. +func TestDB_SubscribePull_until(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make(map[uint8][]storage.Address) + var addrsMu sync.Mutex + var wantedChunksCount int + + lastTimestamp := time.Now().UTC().UnixNano() + var lastTimestampMu sync.RWMutex + defer setNow(func() (t int64) { + lastTimestampMu.Lock() + defer lastTimestampMu.Unlock() + lastTimestamp++ + return lastTimestamp + })() + + uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + last = make(map[uint8]ChunkDescriptor) + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + bin := db.po(chunk.Address()) + + addrsMu.Lock() + if _, ok := addrs[bin]; !ok { + addrs[bin] = make([]storage.Address, 0) + } + if wanted { + addrs[bin] = append(addrs[bin], chunk.Address()) + wantedChunksCount++ + } + addrsMu.Unlock() + + lastTimestampMu.RLock() + storeTimestamp := lastTimestamp + lastTimestampMu.RUnlock() + + last[bin] = ChunkDescriptor{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + } + } + return last + } + + // prepopulate database with some chunks + // before the subscription + last := uploadRandomChunks(30, true) + + uploadRandomChunks(25, false) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + until, ok := last[bin] + if !ok { + continue + } + ch, stop := db.SubscribePull(ctx, bin, nil, &until) + defer stop() + + // receive and validate addresses from the subscription + go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan) + } + + // upload some chunks just after subscribe + uploadRandomChunks(15, false) + + checkErrChan(ctx, t, errChan, wantedChunksCount) +} + +// TestDB_SubscribePull_sinceAndUntil uploads chunks before and +// after pull syncing subscriptions are created with since +// and until arguments, and validates if all expected addresses +// are received in the right order for expected proximity order bins. +func TestDB_SubscribePull_sinceAndUntil(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make(map[uint8][]storage.Address) + var addrsMu sync.Mutex + var wantedChunksCount int + + lastTimestamp := time.Now().UTC().UnixNano() + var lastTimestampMu sync.RWMutex + defer setNow(func() (t int64) { + lastTimestampMu.Lock() + defer lastTimestampMu.Unlock() + lastTimestamp++ + return lastTimestamp + })() + + uploadRandomChunks := func(count int, wanted bool) (last map[uint8]ChunkDescriptor) { + last = make(map[uint8]ChunkDescriptor) + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + bin := db.po(chunk.Address()) + + addrsMu.Lock() + if _, ok := addrs[bin]; !ok { + addrs[bin] = make([]storage.Address, 0) + } + if wanted { + addrs[bin] = append(addrs[bin], chunk.Address()) + wantedChunksCount++ + } + addrsMu.Unlock() + + lastTimestampMu.RLock() + storeTimestamp := lastTimestamp + lastTimestampMu.RUnlock() + + last[bin] = ChunkDescriptor{ + Address: chunk.Address(), + StoreTimestamp: storeTimestamp, + } + } + return last + } + + // all chunks from upload1 are not expected + // as upload1 chunk is used as since for subscriptions + upload1 := uploadRandomChunks(100, false) + + // all chunks from upload2 are expected + // as upload2 chunk is used as until for subscriptions + upload2 := uploadRandomChunks(100, true) + + // upload some chunks before subscribe but after + // wanted chunks + uploadRandomChunks(8, false) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + for bin := uint8(0); bin <= uint8(storage.MaxPO); bin++ { + var since *ChunkDescriptor + if c, ok := upload1[bin]; ok { + since = &c + } + until, ok := upload2[bin] + if !ok { + // no chunks un this bin uploaded in the upload2 + // skip this bin from testing + continue + } + ch, stop := db.SubscribePull(ctx, bin, since, &until) + defer stop() + + // receive and validate addresses from the subscription + go readPullSubscriptionBin(ctx, bin, ch, addrs, &addrsMu, errChan) + } + + // upload some chunks just after subscribe + uploadRandomChunks(15, false) + + checkErrChan(ctx, t, errChan, wantedChunksCount) +} + +// uploadRandomChunksBin uploads random chunks to database and adds them to +// the map of addresses ber bin. +func uploadRandomChunksBin(t *testing.T, db *DB, uploader *Putter, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, wantedChunksCount *int, count int) { + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + addrsMu.Lock() + bin := db.po(chunk.Address()) + if _, ok := addrs[bin]; !ok { + addrs[bin] = make([]storage.Address, 0) + } + addrs[bin] = append(addrs[bin], chunk.Address()) + addrsMu.Unlock() + + *wantedChunksCount++ + } +} + +// readPullSubscriptionBin is a helper function that reads all ChunkDescriptors from a channel and +// sends error to errChan, even if it is nil, to count the number of ChunkDescriptors +// returned by the channel. +func readPullSubscriptionBin(ctx context.Context, bin uint8, ch <-chan ChunkDescriptor, addrs map[uint8][]storage.Address, addrsMu *sync.Mutex, errChan chan error) { + var i int // address index + for { + select { + case got, ok := <-ch: + if !ok { + return + } + addrsMu.Lock() + if i+1 > len(addrs[bin]) { + errChan <- fmt.Errorf("got more chunk addresses %v, then expected %v, for bin %v", i+1, len(addrs[bin]), bin) + } + want := addrs[bin][i] + addrsMu.Unlock() + var err error + if !bytes.Equal(got.Address, want) { + err = fmt.Errorf("got chunk address %v in bin %v %s, want %s", i, bin, got.Address.Hex(), want) + } + i++ + // send one and only one error per received address + errChan <- err + case <-ctx.Done(): + return + } + } +} + +// checkErrChan expects the number of wantedChunksCount errors from errChan +// and calls t.Error for the ones that are not nil. +func checkErrChan(ctx context.Context, t *testing.T, errChan chan error, wantedChunksCount int) { + t.Helper() + + for i := 0; i < wantedChunksCount; i++ { + select { + case err := <-errChan: + if err != nil { + t.Error(err) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } +} diff --git a/swarm/storage/localstore/subscription_push.go b/swarm/storage/localstore/subscription_push.go new file mode 100644 index 0000000000..b13f293998 --- /dev/null +++ b/swarm/storage/localstore/subscription_push.go @@ -0,0 +1,145 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "context" + "sync" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/swarm/shed" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// SubscribePush returns a channel that provides storage chunks with ordering from push syncing index. +// Returned stop function will terminate current and further iterations, and also it will close +// the returned channel without any errors. Make sure that you check the second returned parameter +// from the channel to stop iteration when its value is false. +func (db *DB) SubscribePush(ctx context.Context) (c <-chan storage.Chunk, stop func()) { + chunks := make(chan storage.Chunk) + trigger := make(chan struct{}, 1) + + db.pushTriggersMu.Lock() + db.pushTriggers = append(db.pushTriggers, trigger) + db.pushTriggersMu.Unlock() + + // send signal for the initial iteration + trigger <- struct{}{} + + stopChan := make(chan struct{}) + var stopChanOnce sync.Once + + go func() { + // close the returned chunkInfo channel at the end to + // signal that the subscription is done + defer close(chunks) + // sinceItem is the Item from which the next iteration + // should start. The first iteration starts from the first Item. + var sinceItem *shed.Item + for { + select { + case <-trigger: + // iterate until: + // - last index Item is reached + // - subscription stop is called + // - context is done + err := db.pushIndex.Iterate(func(item shed.Item) (stop bool, err error) { + // get chunk data + dataItem, err := db.retrievalDataIndex.Get(item) + if err != nil { + return true, err + } + + select { + case chunks <- storage.NewChunk(dataItem.Address, dataItem.Data): + // set next iteration start item + // when its chunk is successfully sent to channel + sinceItem = &item + return false, nil + case <-stopChan: + // gracefully stop the iteration + // on stop + return true, nil + case <-db.close: + // gracefully stop the iteration + // on database close + return true, nil + case <-ctx.Done(): + return true, ctx.Err() + } + }, &shed.IterateOptions{ + StartFrom: sinceItem, + // sinceItem was sent as the last Address in the previous + // iterator call, skip it in this one + SkipStartFromItem: true, + }) + if err != nil { + log.Error("localstore push subscription iteration", "err", err) + return + } + case <-stopChan: + // terminate the subscription + // on stop + return + case <-db.close: + // terminate the subscription + // on database close + return + case <-ctx.Done(): + err := ctx.Err() + if err != nil { + log.Error("localstore push subscription", "err", err) + } + return + } + } + }() + + stop = func() { + stopChanOnce.Do(func() { + close(stopChan) + }) + + db.pushTriggersMu.Lock() + defer db.pushTriggersMu.Unlock() + + for i, t := range db.pushTriggers { + if t == trigger { + db.pushTriggers = append(db.pushTriggers[:i], db.pushTriggers[i+1:]...) + break + } + } + } + + return chunks, stop +} + +// triggerPushSubscriptions is used internally for starting iterations +// on Push subscriptions. Whenever new item is added to the push index, +// this function should be called. +func (db *DB) triggerPushSubscriptions() { + db.pushTriggersMu.RLock() + triggers := db.pushTriggers + db.pushTriggersMu.RUnlock() + + for _, t := range triggers { + select { + case t <- struct{}{}: + default: + } + } +} diff --git a/swarm/storage/localstore/subscription_push_test.go b/swarm/storage/localstore/subscription_push_test.go new file mode 100644 index 0000000000..73e7c25f72 --- /dev/null +++ b/swarm/storage/localstore/subscription_push_test.go @@ -0,0 +1,200 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package localstore + +import ( + "bytes" + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" +) + +// TestDB_SubscribePush uploads some chunks before and after +// push syncing subscription is created and validates if +// all addresses are received in the right order. +func TestDB_SubscribePush(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + chunks := make([]storage.Chunk, 0) + var chunksMu sync.Mutex + + uploadRandomChunks := func(count int) { + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + chunksMu.Lock() + chunks = append(chunks, chunk) + chunksMu.Unlock() + } + } + + // prepopulate database with some chunks + // before the subscription + uploadRandomChunks(10) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + ch, stop := db.SubscribePush(ctx) + defer stop() + + // receive and validate addresses from the subscription + go func() { + var i int // address index + for { + select { + case got, ok := <-ch: + if !ok { + return + } + chunksMu.Lock() + want := chunks[i] + chunksMu.Unlock() + var err error + if !bytes.Equal(got.Data(), want.Data()) { + err = fmt.Errorf("got chunk %v data %x, want %x", i, got.Data(), want.Data()) + } + if !bytes.Equal(got.Address(), want.Address()) { + err = fmt.Errorf("got chunk %v address %s, want %s", i, got.Address().Hex(), want.Address().Hex()) + } + i++ + // send one and only one error per received address + errChan <- err + case <-ctx.Done(): + return + } + } + }() + + // upload some chunks just after subscribe + uploadRandomChunks(5) + + time.Sleep(200 * time.Millisecond) + + // upload some chunks after some short time + // to ensure that subscription will include them + // in a dynamic environment + uploadRandomChunks(3) + + checkErrChan(ctx, t, errChan, len(chunks)) +} + +// TestDB_SubscribePush_multiple uploads chunks before and after +// multiple push syncing subscriptions are created and +// validates if all addresses are received in the right order. +func TestDB_SubscribePush_multiple(t *testing.T) { + db, cleanupFunc := newTestDB(t, nil) + defer cleanupFunc() + + uploader := db.NewPutter(ModePutUpload) + + addrs := make([]storage.Address, 0) + var addrsMu sync.Mutex + + uploadRandomChunks := func(count int) { + for i := 0; i < count; i++ { + chunk := generateRandomChunk() + + err := uploader.Put(chunk) + if err != nil { + t.Fatal(err) + } + + addrsMu.Lock() + addrs = append(addrs, chunk.Address()) + addrsMu.Unlock() + } + } + + // prepopulate database with some chunks + // before the subscription + uploadRandomChunks(10) + + // set a timeout on subscription + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // collect all errors from validating addresses, even nil ones + // to validate the number of addresses received by the subscription + errChan := make(chan error) + + subsCount := 10 + + // start a number of subscriptions + // that all of them will write every addresses error to errChan + for j := 0; j < subsCount; j++ { + ch, stop := db.SubscribePush(ctx) + defer stop() + + // receive and validate addresses from the subscription + go func(j int) { + var i int // address index + for { + select { + case got, ok := <-ch: + if !ok { + return + } + addrsMu.Lock() + want := addrs[i] + addrsMu.Unlock() + var err error + if !bytes.Equal(got.Address(), want) { + err = fmt.Errorf("got chunk %v address on subscription %v %s, want %s", i, j, got, want) + } + i++ + // send one and only one error per received address + errChan <- err + case <-ctx.Done(): + return + } + } + }(j) + } + + // upload some chunks just after subscribe + uploadRandomChunks(5) + + time.Sleep(200 * time.Millisecond) + + // upload some chunks after some short time + // to ensure that subscription will include them + // in a dynamic environment + uploadRandomChunks(3) + + // number of addresses received by all subscriptions + wantedChunksCount := len(addrs) * subsCount + + checkErrChan(ctx, t, errChan, wantedChunksCount) +} From da1efdae0c2e6bb221927937eac0ea76225d97c2 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 4 Feb 2019 11:54:39 +0100 Subject: [PATCH 06/22] core: repro #18977 --- core/blockchain_test.go | 48 ++++++++++++++++++++++++++++++++++ core/chain_makers.go | 57 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index cfc44bf794..ee3a149417 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1483,3 +1483,51 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { benchmarkLargeNumberOfValueToNonexisting(b, numTxs, numBlocks, recipientFn, dataFn) } + +// Tests that importing a very large side fork, which is larger than the canon chain, +// but where the difficulty per block is kept low: this means that it will not +// overtake the 'canon' chain until after it's passed canon by about 200 blocks. +func TestLargeOldSidechainWithALowTdChain(t *testing.T) { + + // Generate a canonical chain to act as the main dataset + engine := ethash.NewFaker() + + db := ethdb.NewMemDatabase() + genesis := new(Genesis).MustCommit(db) + // We must use a pretty long chain to ensure that the fork + // doesn't overtake us until after at least 128 blocks post tip + blocks, _ := generateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, makeHeaderWithLargeDifficulty) + + // Import the canonical chain + diskdb := ethdb.NewMemDatabase() + new(Genesis).MustCommit(diskdb) + + chain, err := NewBlockChain(diskdb, nil, params.TestChainConfig, engine, vm.Config{}, nil) + if err != nil { + t.Fatalf("failed to create tester chain: %v", err) + } + for i := 0; i < len(blocks); i++ { + if _, err := chain.InsertChain(blocks[i : i+1]); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", i, err) + } + } + // Dereference all the recent tries and ensure no past trie is left in + for i := 0; i < triesInMemory; i++ { + chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) + } + + // Generate fork chain, starting from an early block + parent := blocks[10] + fork, _ := generateChain(params.TestChainConfig, parent, engine, db, 256+6*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }, makeHeaderWithSmallDifficulty) + + // And now import the fork + if i, err := chain.InsertChain(fork); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", i, err) + } + head := chain.CurrentBlock() + if got := fork[len(fork)-1].Hash(); got != head.Hash() { + t.Fatalf("head wrong, expected %x got %x", head.Hash(), got) + } + td := chain.GetTd(head.Hash(), head.NumberU64()) + fmt.Printf("td %v", td) +} diff --git a/core/chain_makers.go b/core/chain_makers.go index e3a5537a40..63ca16440b 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -48,6 +48,8 @@ type BlockGen struct { engine consensus.Engine } +type headerGenFn func(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header + // SetCoinbase sets the coinbase of the generated block. // It can be called at most once. func (b *BlockGen) SetCoinbase(addr common.Address) { @@ -170,6 +172,10 @@ func (b *BlockGen) OffsetTime(seconds int64) { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { + return generateChain(config, parent, engine, db, n, gen, makeHeader) +} + +func generateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen), headerGen headerGenFn) ([]*types.Block, []types.Receipts) { if config == nil { config = params.TestChainConfig } @@ -177,7 +183,8 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} - b.header = makeHeader(chainreader, parent, statedb, b.engine) + //b.header = makeHeader(chainreader, parent, statedb, b.engine) + b.header = headerGen(chainreader, parent, statedb, b.engine) // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { @@ -248,6 +255,54 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S } } +func makeHeaderWithLargeDifficulty(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { + var time *big.Int + if parent.Time() == nil { + time = big.NewInt(1) + } else { + time = new(big.Int).Add(parent.Time(), big.NewInt(1)) // block time is fixed at 10 seconds + } + + return &types.Header{ + Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), + ParentHash: parent.Hash(), + Coinbase: parent.Coinbase(), + Difficulty: engine.CalcDifficulty(chain, time.Uint64(), &types.Header{ + Number: parent.Number(), + Time: new(big.Int).Sub(time, big.NewInt(1)), + Difficulty: parent.Difficulty(), + UncleHash: parent.UncleHash(), + }), + GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), + Number: new(big.Int).Add(parent.Number(), common.Big1), + Time: time, + } +} + +func makeHeaderWithSmallDifficulty(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { + var time *big.Int + if parent.Time() == nil { + time = big.NewInt(30) + } else { + time = new(big.Int).Add(parent.Time(), big.NewInt(30)) // block time is fixed at 10 seconds + } + + return &types.Header{ + Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), + ParentHash: parent.Hash(), + Coinbase: parent.Coinbase(), + Difficulty: engine.CalcDifficulty(chain, time.Uint64(), &types.Header{ + Number: parent.Number(), + Time: new(big.Int).Sub(time, big.NewInt(30)), + Difficulty: parent.Difficulty(), + UncleHash: parent.UncleHash(), + }), + GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), + Number: new(big.Int).Add(parent.Number(), common.Big1), + Time: time, + } +} + // makeHeaderChain creates a deterministic chain of headers rooted at parent. func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Header { blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, engine, db, seed) From 940e3170949be2e2eec389fcc812f23342da4622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Feb 2019 11:11:31 +0200 Subject: [PATCH 07/22] core: fix pruner panic when importing low-diff-large-sidechain --- core/blockchain.go | 28 +++++++++++-------- core/blockchain_test.go | 43 +++++++++++++++++------------- core/chain_makers.go | 59 ++--------------------------------------- 3 files changed, 44 insertions(+), 86 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index c852926ef4..feb9fb9426 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -979,20 +979,26 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - header := bc.GetHeaderByNumber(current - triesInMemory) - chosen := header.Number.Uint64() + chosen := current - triesInMemory // If we exceeded out time allowance, flush an entire trie to disk if bc.gcproc > bc.cacheConfig.TrieTimeLimit { - // If we're exceeding limits but haven't reached a large enough memory gap, - // warn the user that the system is becoming unstable. - if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory) + // If the header is missing (canonical chain behind), we're reorging a low + // diff sidechain. Suspend committing until this operation is completed. + header := bc.GetHeaderByNumber(chosen) + if header == nil { + log.Warn("Reorg in progress, trie commit postponed", "number", chosen) + } else { + // If we're exceeding limits but haven't reached a large enough memory gap, + // warn the user that the system is becoming unstable. + if chosen < lastWrite+triesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/triesInMemory) + } + // Flush an entire trie and restart the counters + triedb.Commit(header.Root, true) + lastWrite = chosen + bc.gcproc = 0 } - // Flush an entire trie and restart the counters - triedb.Commit(header.Root, true) - lastWrite = chosen - bc.gcproc = 0 } // Garbage collect anything below our required write retention for !bc.triegc.Empty() { @@ -1324,7 +1330,7 @@ func (bc *BlockChain) insertSidechain(block *types.Block, it *insertIterator) (i if err := bc.WriteBlockWithoutState(block, externTd); err != nil { return it.index, nil, nil, err } - log.Debug("Inserted sidechain block", "number", block.Number(), "hash", block.Hash(), + log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index ee3a149417..fa68c2894a 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1487,16 +1487,22 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // Tests that importing a very large side fork, which is larger than the canon chain, // but where the difficulty per block is kept low: this means that it will not // overtake the 'canon' chain until after it's passed canon by about 200 blocks. -func TestLargeOldSidechainWithALowTdChain(t *testing.T) { - +// +// Details at: +// - https://github.com/ethereum/go-ethereum/issues/18977 +// - https://github.com/ethereum/go-ethereum/pull/18988 +func TestLowDiffLongChain(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() - db := ethdb.NewMemDatabase() genesis := new(Genesis).MustCommit(db) - // We must use a pretty long chain to ensure that the fork - // doesn't overtake us until after at least 128 blocks post tip - blocks, _ := generateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }, makeHeaderWithLargeDifficulty) + + // We must use a pretty long chain to ensure that the fork doesn't overtake us + // until after at least 128 blocks post tip + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*triesInMemory, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{1}) + b.OffsetTime(-9) + }) // Import the canonical chain diskdb := ethdb.NewMemDatabase() @@ -1506,19 +1512,14 @@ func TestLargeOldSidechainWithALowTdChain(t *testing.T) { if err != nil { t.Fatalf("failed to create tester chain: %v", err) } - for i := 0; i < len(blocks); i++ { - if _, err := chain.InsertChain(blocks[i : i+1]); err != nil { - t.Fatalf("block %d: failed to insert into chain: %v", i, err) - } + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - // Dereference all the recent tries and ensure no past trie is left in - for i := 0; i < triesInMemory; i++ { - chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) - } - // Generate fork chain, starting from an early block parent := blocks[10] - fork, _ := generateChain(params.TestChainConfig, parent, engine, db, 256+6*triesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }, makeHeaderWithSmallDifficulty) + fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*triesInMemory, func(i int, b *BlockGen) { + b.SetCoinbase(common.Address{2}) + }) // And now import the fork if i, err := chain.InsertChain(fork); err != nil { @@ -1528,6 +1529,12 @@ func TestLargeOldSidechainWithALowTdChain(t *testing.T) { if got := fork[len(fork)-1].Hash(); got != head.Hash() { t.Fatalf("head wrong, expected %x got %x", head.Hash(), got) } - td := chain.GetTd(head.Hash(), head.NumberU64()) - fmt.Printf("td %v", td) + // Sanity check that all the canonical numbers are present + header := chain.CurrentHeader() + for number := head.NumberU64(); number > 0; number-- { + if hash := chain.GetHeaderByNumber(number).Hash(); hash != header.Hash() { + t.Fatalf("header %d: canonical hash mismatch: have %x, want %x", number, hash, header.Hash()) + } + header = chain.GetHeader(header.ParentHash, number-1) + } } diff --git a/core/chain_makers.go b/core/chain_makers.go index 63ca16440b..0b5a3d1843 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -48,8 +48,6 @@ type BlockGen struct { engine consensus.Engine } -type headerGenFn func(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header - // SetCoinbase sets the coinbase of the generated block. // It can be called at most once. func (b *BlockGen) SetCoinbase(addr common.Address) { @@ -151,7 +149,7 @@ func (b *BlockGen) PrevBlock(index int) *types.Block { // associated difficulty. It's useful to test scenarios where forking is not // tied to chain length directly. func (b *BlockGen) OffsetTime(seconds int64) { - b.header.Time.Add(b.header.Time, new(big.Int).SetInt64(seconds)) + b.header.Time.Add(b.header.Time, big.NewInt(seconds)) if b.header.Time.Cmp(b.parent.Header().Time) <= 0 { panic("block time out of range") } @@ -172,10 +170,6 @@ func (b *BlockGen) OffsetTime(seconds int64) { // values. Inserting them into BlockChain requires use of FakePow or // a similar non-validating proof of work implementation. func GenerateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) { - return generateChain(config, parent, engine, db, n, gen, makeHeader) -} - -func generateChain(config *params.ChainConfig, parent *types.Block, engine consensus.Engine, db ethdb.Database, n int, gen func(int, *BlockGen), headerGen headerGenFn) ([]*types.Block, []types.Receipts) { if config == nil { config = params.TestChainConfig } @@ -183,8 +177,7 @@ func generateChain(config *params.ChainConfig, parent *types.Block, engine conse chainreader := &fakeChainReader{config: config} genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) { b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine} - //b.header = makeHeader(chainreader, parent, statedb, b.engine) - b.header = headerGen(chainreader, parent, statedb, b.engine) + b.header = makeHeader(chainreader, parent, statedb, b.engine) // Mutate the state and block according to any hard-fork specs if daoBlock := config.DAOForkBlock; daoBlock != nil { @@ -255,54 +248,6 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S } } -func makeHeaderWithLargeDifficulty(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { - var time *big.Int - if parent.Time() == nil { - time = big.NewInt(1) - } else { - time = new(big.Int).Add(parent.Time(), big.NewInt(1)) // block time is fixed at 10 seconds - } - - return &types.Header{ - Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), - ParentHash: parent.Hash(), - Coinbase: parent.Coinbase(), - Difficulty: engine.CalcDifficulty(chain, time.Uint64(), &types.Header{ - Number: parent.Number(), - Time: new(big.Int).Sub(time, big.NewInt(1)), - Difficulty: parent.Difficulty(), - UncleHash: parent.UncleHash(), - }), - GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), - Number: new(big.Int).Add(parent.Number(), common.Big1), - Time: time, - } -} - -func makeHeaderWithSmallDifficulty(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { - var time *big.Int - if parent.Time() == nil { - time = big.NewInt(30) - } else { - time = new(big.Int).Add(parent.Time(), big.NewInt(30)) // block time is fixed at 10 seconds - } - - return &types.Header{ - Root: state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())), - ParentHash: parent.Hash(), - Coinbase: parent.Coinbase(), - Difficulty: engine.CalcDifficulty(chain, time.Uint64(), &types.Header{ - Number: parent.Number(), - Time: new(big.Int).Sub(time, big.NewInt(30)), - Difficulty: parent.Difficulty(), - UncleHash: parent.UncleHash(), - }), - GasLimit: CalcGasLimit(parent, parent.GasLimit(), parent.GasLimit()), - Number: new(big.Int).Add(parent.Number(), common.Big1), - Time: time, - } -} - // makeHeaderChain creates a deterministic chain of headers rooted at parent. func makeHeaderChain(parent *types.Header, n int, engine consensus.Engine, db ethdb.Database, seed int) []*types.Header { blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, engine, db, seed) From 0c10d376066cb7e57d3bfc03f950c7750cd90640 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 8 Feb 2019 16:57:48 +0100 Subject: [PATCH 08/22] swarm/network, swarm/storage: Preserve opentracing contexts (#19022) --- swarm/network/fetcher.go | 36 +++++----- swarm/network/fetcher_test.go | 115 +++++++++++++++---------------- swarm/network/stream/delivery.go | 7 +- swarm/network/stream/messages.go | 2 +- swarm/network/stream/peer.go | 26 ++++--- swarm/network/stream/stream.go | 5 +- swarm/storage/chunker.go | 16 +++-- swarm/storage/feed/testutil.go | 4 +- swarm/storage/netstore.go | 16 ++--- swarm/storage/netstore_test.go | 4 +- 10 files changed, 123 insertions(+), 108 deletions(-) diff --git a/swarm/network/fetcher.go b/swarm/network/fetcher.go index 3043f64758..6b21751669 100644 --- a/swarm/network/fetcher.go +++ b/swarm/network/fetcher.go @@ -52,6 +52,7 @@ type Fetcher struct { requestC chan uint8 // channel for incoming requests (with the hopCount value in it) searchTimeout time.Duration skipCheck bool + ctx context.Context } type Request struct { @@ -109,14 +110,14 @@ func NewFetcherFactory(request RequestFunc, skipCheck bool) *FetcherFactory { // contain the peers which are actively requesting this chunk, to make sure we // don't request back the chunks from them. // The created Fetcher is started and returned. -func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peersToSkip *sync.Map) storage.NetFetcher { - fetcher := NewFetcher(source, f.request, f.skipCheck) - go fetcher.run(ctx, peersToSkip) +func (f *FetcherFactory) New(ctx context.Context, source storage.Address, peers *sync.Map) storage.NetFetcher { + fetcher := NewFetcher(ctx, source, f.request, f.skipCheck) + go fetcher.run(peers) return fetcher } // NewFetcher creates a new Fetcher for the given chunk address using the given request function. -func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { +func NewFetcher(ctx context.Context, addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { return &Fetcher{ addr: addr, protoRequestFunc: rf, @@ -124,14 +125,15 @@ func NewFetcher(addr storage.Address, rf RequestFunc, skipCheck bool) *Fetcher { requestC: make(chan uint8), searchTimeout: defaultSearchTimeout, skipCheck: skipCheck, + ctx: ctx, } } // Offer is called when an upstream peer offers the chunk via syncing as part of `OfferedHashesMsg` and the node does not have the chunk locally. -func (f *Fetcher) Offer(ctx context.Context, source *enode.ID) { +func (f *Fetcher) Offer(source *enode.ID) { // First we need to have this select to make sure that we return if context is done select { - case <-ctx.Done(): + case <-f.ctx.Done(): return default: } @@ -140,15 +142,15 @@ func (f *Fetcher) Offer(ctx context.Context, source *enode.ID) { // push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements) select { case f.offerC <- source: - case <-ctx.Done(): + case <-f.ctx.Done(): } } // Request is called when an upstream peer request the chunk as part of `RetrieveRequestMsg`, or from a local request through FileStore, and the node does not have the chunk locally. -func (f *Fetcher) Request(ctx context.Context, hopCount uint8) { +func (f *Fetcher) Request(hopCount uint8) { // First we need to have this select to make sure that we return if context is done select { - case <-ctx.Done(): + case <-f.ctx.Done(): return default: } @@ -162,13 +164,13 @@ func (f *Fetcher) Request(ctx context.Context, hopCount uint8) { // push to offerC instead if offerC is available (see number 2 in https://golang.org/ref/spec#Select_statements) select { case f.requestC <- hopCount + 1: - case <-ctx.Done(): + case <-f.ctx.Done(): } } // start prepares the Fetcher // it keeps the Fetcher alive within the lifecycle of the passed context -func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { +func (f *Fetcher) run(peers *sync.Map) { var ( doRequest bool // determines if retrieval is initiated in the current iteration wait *time.Timer // timer for search timeout @@ -219,7 +221,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { doRequest = requested // all Fetcher context closed, can quit - case <-ctx.Done(): + case <-f.ctx.Done(): log.Trace("terminate fetcher", "request addr", f.addr) // TODO: send cancellations to all peers left over in peers map (i.e., those we requested from) return @@ -228,7 +230,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // need to issue a new request if doRequest { var err error - sources, err = f.doRequest(ctx, gone, peers, sources, hopCount) + sources, err = f.doRequest(gone, peers, sources, hopCount) if err != nil { log.Info("unable to request", "request addr", f.addr, "err", err) } @@ -266,7 +268,7 @@ func (f *Fetcher) run(ctx context.Context, peers *sync.Map) { // * the peer's address is added to the set of peers to skip // * the peer's address is removed from prospective sources, and // * a go routine is started that reports on the gone channel if the peer is disconnected (or terminated their streamer) -func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID, hopCount uint8) ([]*enode.ID, error) { +func (f *Fetcher) doRequest(gone chan *enode.ID, peersToSkip *sync.Map, sources []*enode.ID, hopCount uint8) ([]*enode.ID, error) { var i int var sourceID *enode.ID var quit chan struct{} @@ -283,7 +285,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki for i = 0; i < len(sources); i++ { req.Source = sources[i] var err error - sourceID, quit, err = f.protoRequestFunc(ctx, req) + sourceID, quit, err = f.protoRequestFunc(f.ctx, req) if err == nil { // remove the peer from known sources // Note: we can modify the source although we are looping on it, because we break from the loop immediately @@ -297,7 +299,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki if !foundSource { req.Source = nil var err error - sourceID, quit, err = f.protoRequestFunc(ctx, req) + sourceID, quit, err = f.protoRequestFunc(f.ctx, req) if err != nil { // if no peers found to request from return sources, err @@ -314,7 +316,7 @@ func (f *Fetcher) doRequest(ctx context.Context, gone chan *enode.ID, peersToSki select { case <-quit: gone <- sourceID - case <-ctx.Done(): + case <-f.ctx.Done(): } }() return sources, nil diff --git a/swarm/network/fetcher_test.go b/swarm/network/fetcher_test.go index 563c6cece7..4e464f10f3 100644 --- a/swarm/network/fetcher_test.go +++ b/swarm/network/fetcher_test.go @@ -69,7 +69,11 @@ func (m *mockRequester) doRequest(ctx context.Context, request *Request) (*enode func TestFetcherSingleRequest(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) peers := []string{"a", "b", "c", "d"} peersToSkip := &sync.Map{} @@ -77,13 +81,9 @@ func TestFetcherSingleRequest(t *testing.T) { peersToSkip.Store(p, time.Now()) } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + go fetcher.run(peersToSkip) - go fetcher.run(ctx, peersToSkip) - - rctx := context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) select { case request := <-requester.requestC: @@ -115,20 +115,19 @@ func TestFetcherSingleRequest(t *testing.T) { func TestFetcherCancelStopsFetcher(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) - - peersToSkip := &sync.Map{} ctx, cancel := context.WithCancel(context.Background()) + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + // we start the fetcher, and then we immediately cancel the context - go fetcher.run(ctx, peersToSkip) + go fetcher.run(peersToSkip) cancel() - rctx, rcancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer rcancel() // we call Request with an active context - fetcher.Request(rctx, 0) + fetcher.Request(0) // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening select { @@ -140,23 +139,23 @@ func TestFetcherCancelStopsFetcher(t *testing.T) { // TestFetchCancelStopsRequest tests that calling a Request function with a cancelled context does not initiate a request func TestFetcherCancelStopsRequest(t *testing.T) { + t.Skip("since context is now per fetcher, this test is likely redundant") + requester := newMockRequester(100 * time.Millisecond) addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) - - peersToSkip := &sync.Map{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // we start the fetcher with an active context - go fetcher.run(ctx, peersToSkip) + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) - rctx, rcancel := context.WithCancel(context.Background()) - rcancel() + peersToSkip := &sync.Map{} + + // we start the fetcher with an active context + go fetcher.run(peersToSkip) // we call Request with a cancelled context - fetcher.Request(rctx, 0) + fetcher.Request(0) // fetcher should not initiate request, we can only check by waiting a bit and making sure no request is happening select { @@ -166,8 +165,7 @@ func TestFetcherCancelStopsRequest(t *testing.T) { } // if there is another Request with active context, there should be a request, because the fetcher itself is not cancelled - rctx = context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) select { case <-requester.requestC: @@ -182,19 +180,19 @@ func TestFetcherCancelStopsRequest(t *testing.T) { func TestFetcherOfferUsesSource(t *testing.T) { requester := newMockRequester(100 * time.Millisecond) addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) - - peersToSkip := &sync.Map{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // start the fetcher - go fetcher.run(ctx, peersToSkip) + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + + // start the fetcher + go fetcher.run(peersToSkip) - rctx := context.Background() // call the Offer function with the source peer - fetcher.Offer(rctx, &sourcePeerID) + fetcher.Offer(&sourcePeerID) // fetcher should not initiate request select { @@ -204,8 +202,7 @@ func TestFetcherOfferUsesSource(t *testing.T) { } // call Request after the Offer - rctx = context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) // there should be exactly 1 request coming from fetcher var request *Request @@ -234,19 +231,19 @@ func TestFetcherOfferUsesSource(t *testing.T) { func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) { requester := newMockRequester(100 * time.Millisecond) addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) - - peersToSkip := &sync.Map{} ctx, cancel := context.WithCancel(context.Background()) defer cancel() + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) + + peersToSkip := &sync.Map{} + // start the fetcher - go fetcher.run(ctx, peersToSkip) + go fetcher.run(peersToSkip) // call Request first - rctx := context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) // there should be a request coming from fetcher var request *Request @@ -260,7 +257,7 @@ func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) { } // after the Request call Offer - fetcher.Offer(context.Background(), &sourcePeerID) + fetcher.Offer(&sourcePeerID) // there should be a request coming from fetcher select { @@ -283,21 +280,21 @@ func TestFetcherOfferAfterRequestUsesSourceFromContext(t *testing.T) { func TestFetcherRetryOnTimeout(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) // set searchTimeOut to low value so the test is quicker fetcher.searchTimeout = 250 * time.Millisecond peersToSkip := &sync.Map{} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // start the fetcher - go fetcher.run(ctx, peersToSkip) + go fetcher.run(peersToSkip) // call the fetch function with an active context - rctx := context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) // after 100ms the first request should be initiated time.Sleep(100 * time.Millisecond) @@ -339,7 +336,7 @@ func TestFetcherFactory(t *testing.T) { fetcher := fetcherFactory.New(context.Background(), addr, peersToSkip) - fetcher.Request(context.Background(), 0) + fetcher.Request(0) // check if the created fetchFunction really starts a fetcher and initiates a request select { @@ -353,7 +350,11 @@ func TestFetcherFactory(t *testing.T) { func TestFetcherRequestQuitRetriesRequest(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) // make sure the searchTimeout is long so it is sure the request is not // retried because of timeout @@ -361,13 +362,9 @@ func TestFetcherRequestQuitRetriesRequest(t *testing.T) { peersToSkip := &sync.Map{} - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + go fetcher.run(peersToSkip) - go fetcher.run(ctx, peersToSkip) - - rctx := context.Background() - fetcher.Request(rctx, 0) + fetcher.Request(0) select { case <-requester.requestC: @@ -460,17 +457,15 @@ func TestRequestSkipPeerPermanent(t *testing.T) { func TestFetcherMaxHopCount(t *testing.T) { requester := newMockRequester() addr := make([]byte, 32) - fetcher := NewFetcher(addr, requester.doRequest, true) ctx, cancel := context.WithCancel(context.Background()) defer cancel() + fetcher := NewFetcher(ctx, addr, requester.doRequest, true) + peersToSkip := &sync.Map{} - go fetcher.run(ctx, peersToSkip) - - rctx := context.Background() - fetcher.Request(rctx, maxHopCount) + go fetcher.run(peersToSkip) // if hopCount is already at max no request should be initiated select { diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index 988afcce84..fae6994f0c 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -144,7 +144,6 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * ctx, osp = spancontext.StartSpan( ctx, "retrieve.request") - defer osp.Finish() s, err := sp.getServer(NewStream(swarmChunkServerStreamName, "", true)) if err != nil { @@ -167,6 +166,7 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * }() go func() { + defer osp.Finish() chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { retrieveChunkFail.Inc(1) @@ -213,11 +213,12 @@ func (d *Delivery) handleChunkDeliveryMsg(ctx context.Context, sp *Peer, req *Ch ctx, osp = spancontext.StartSpan( ctx, "chunk.delivery") - defer osp.Finish() processReceivedChunksCount.Inc(1) go func() { + defer osp.Finish() + req.peer = sp err := d.chunkStore.Put(ctx, storage.NewChunk(req.Addr, req.SData)) if err != nil { @@ -271,7 +272,7 @@ func (d *Delivery) RequestFromPeers(ctx context.Context, req *network.Request) ( Addr: req.Addr, SkipCheck: req.SkipCheck, HopCount: req.HopCount, - }, Top) + }, Top, "request.from.peers") if err != nil { return nil, nil, err } diff --git a/swarm/network/stream/messages.go b/swarm/network/stream/messages.go index b293724cc7..de4e8a3bb4 100644 --- a/swarm/network/stream/messages.go +++ b/swarm/network/stream/messages.go @@ -300,7 +300,7 @@ func (p *Peer) handleOfferedHashesMsg(ctx context.Context, req *OfferedHashesMsg return } log.Trace("sending want batch", "peer", p.ID(), "stream", msg.Stream, "from", msg.From, "to", msg.To) - err := p.SendPriority(ctx, msg, c.priority) + err := p.SendPriority(ctx, msg, c.priority, "") if err != nil { log.Warn("SendPriority error", "err", err) } diff --git a/swarm/network/stream/peer.go b/swarm/network/stream/peer.go index 4bccf56f5d..68da8f44ad 100644 --- a/swarm/network/stream/peer.go +++ b/swarm/network/stream/peer.go @@ -65,6 +65,7 @@ type Peer struct { // on creating a new client in offered hashes handler. clientParams map[Stream]*clientParams quit chan struct{} + spans sync.Map } type WrappedPriorityMsg struct { @@ -82,10 +83,16 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { clients: make(map[Stream]*client), clientParams: make(map[Stream]*clientParams), quit: make(chan struct{}), + spans: sync.Map{}, } ctx, cancel := context.WithCancel(context.Background()) go p.pq.Run(ctx, func(i interface{}) { wmsg := i.(WrappedPriorityMsg) + defer p.spans.Delete(wmsg.Context) + sp, ok := p.spans.Load(wmsg.Context) + if ok { + defer sp.(opentracing.Span).Finish() + } err := p.Send(wmsg.Context, wmsg.Msg) if err != nil { log.Error("Message send error, dropping peer", "peer", p.ID(), "err", err) @@ -130,7 +137,6 @@ func NewPeer(peer *protocols.Peer, streamer *Registry) *Peer { // Deliver sends a storeRequestMsg protocol message to the peer // Depending on the `syncing` parameter we send different message types func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, syncing bool) error { - var sp opentracing.Span var msg interface{} spanName := "send.chunk.delivery" @@ -151,18 +157,22 @@ func (p *Peer) Deliver(ctx context.Context, chunk storage.Chunk, priority uint8, } spanName += ".retrieval" } - ctx, sp = spancontext.StartSpan( - ctx, - spanName) - defer sp.Finish() - return p.SendPriority(ctx, msg, priority) + return p.SendPriority(ctx, msg, priority, spanName) } // SendPriority sends message to the peer using the outgoing priority queue -func (p *Peer) SendPriority(ctx context.Context, msg interface{}, priority uint8) error { +func (p *Peer) SendPriority(ctx context.Context, msg interface{}, priority uint8, traceId string) error { defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("peer.sendpriority_t.%d", priority), nil).UpdateSince(time.Now()) metrics.GetOrRegisterCounter(fmt.Sprintf("peer.sendpriority.%d", priority), nil).Inc(1) + if traceId != "" { + var sp opentracing.Span + ctx, sp = spancontext.StartSpan( + ctx, + traceId, + ) + p.spans.Store(ctx, sp) + } wmsg := WrappedPriorityMsg{ Context: ctx, Msg: msg, @@ -205,7 +215,7 @@ func (p *Peer) SendOfferedHashes(s *server, f, t uint64) error { Stream: s.stream, } log.Trace("Swarm syncer offer batch", "peer", p.ID(), "stream", s.stream, "len", len(hashes), "from", from, "to", to) - return p.SendPriority(ctx, msg, s.priority) + return p.SendPriority(ctx, msg, s.priority, "send.offered.hashes") } func (p *Peer) getServer(s Stream) (*server, error) { diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 485a69ea28..5d3e23eb18 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -359,7 +359,7 @@ func (r *Registry) Subscribe(peerId enode.ID, s Stream, h *Range, priority uint8 } log.Debug("Subscribe ", "peer", peerId, "stream", s, "history", h) - return peer.SendPriority(context.TODO(), msg, priority) + return peer.SendPriority(context.TODO(), msg, priority, "") } func (r *Registry) Unsubscribe(peerId enode.ID, s Stream) error { @@ -729,7 +729,8 @@ func (c *client) batchDone(p *Peer, req *OfferedHashesMsg, hashes []byte) error if err != nil { return err } - if err := p.SendPriority(context.TODO(), tp, c.priority); err != nil { + + if err := p.SendPriority(context.TODO(), tp, c.priority, ""); err != nil { return err } if c.to > 0 && tp.Takeover.End >= c.to { diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index a8bfe2d1c9..0fa5026dcd 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -465,7 +465,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { length *= r.chunkSize } wg.Add(1) - go r.join(b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC) + go r.join(cctx, b, off, off+length, depth, treeSize/r.branches, r.chunkData, &wg, errC, quitC) go func() { wg.Wait() close(errC) @@ -485,7 +485,7 @@ func (r *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { return len(b), nil } -func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { +func (r *LazyChunkReader) join(ctx context.Context, b []byte, off int64, eoff int64, depth int, treeSize int64, chunkData ChunkData, parentWg *sync.WaitGroup, errC chan error, quitC chan bool) { defer parentWg.Done() // find appropriate block level for chunkData.Size() < uint64(treeSize) && depth > r.depth { @@ -533,7 +533,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS go func(j int64) { childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] startTime := time.Now() - chunkData, err := r.getter.Get(r.ctx, Reference(childAddress)) + chunkData, err := r.getter.Get(ctx, Reference(childAddress)) if err != nil { metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err) @@ -554,7 +554,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS if soff < off { soff = off } - r.join(b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC) + r.join(ctx, b[soff-off:seoff-off], soff-roff, seoff-roff, depth-1, treeSize/r.branches, chunkData, wg, errC, quitC) }(i) } //for } @@ -581,6 +581,11 @@ var errWhence = errors.New("Seek: invalid whence") var errOffset = errors.New("Seek: invalid offset") func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { + cctx, sp := spancontext.StartSpan( + r.ctx, + "lcr.seek") + defer sp.Finish() + log.Debug("lazychunkreader.seek", "key", r.addr, "offset", offset) switch whence { default: @@ -590,8 +595,9 @@ func (r *LazyChunkReader) Seek(offset int64, whence int) (int64, error) { case 1: offset += r.off case 2: + if r.chunkData == nil { //seek from the end requires rootchunk for size. call Size first - _, err := r.Size(context.TODO(), nil) + _, err := r.Size(cctx, nil) if err != nil { return 0, fmt.Errorf("can't get size: %v", err) } diff --git a/swarm/storage/feed/testutil.go b/swarm/storage/feed/testutil.go index b513fa1f2f..caa39d9ffc 100644 --- a/swarm/storage/feed/testutil.go +++ b/swarm/storage/feed/testutil.go @@ -40,9 +40,9 @@ func (t *TestHandler) Close() { type mockNetFetcher struct{} -func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) { +func (m *mockNetFetcher) Request(hopCount uint8) { } -func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { +func (m *mockNetFetcher) Offer(source *enode.ID) { } func newFakeNetFetcher(context.Context, storage.Address, *sync.Map) storage.NetFetcher { diff --git a/swarm/storage/netstore.go b/swarm/storage/netstore.go index b24d08bc2d..a2595d9fad 100644 --- a/swarm/storage/netstore.go +++ b/swarm/storage/netstore.go @@ -34,8 +34,8 @@ type ( ) type NetFetcher interface { - Request(ctx context.Context, hopCount uint8) - Offer(ctx context.Context, source *enode.ID) + Request(hopCount uint8) + Offer(source *enode.ID) } // NetStore is an extension of local storage @@ -150,7 +150,7 @@ func (n *NetStore) get(ctx context.Context, ref Address) (Chunk, func(context.Co } // The chunk is not available in the LocalStore, let's get the fetcher for it, or create a new one // if it doesn't exist yet - f := n.getOrCreateFetcher(ref) + f := n.getOrCreateFetcher(ctx, ref) // If the caller needs the chunk, it has to use the returned fetch function to get it return nil, f.Fetch, nil } @@ -168,7 +168,7 @@ func (n *NetStore) Has(ctx context.Context, ref Address) bool { // getOrCreateFetcher attempts at retrieving an existing fetchers // if none exists, creates one and saves it in the fetchers cache // caller must hold the lock -func (n *NetStore) getOrCreateFetcher(ref Address) *fetcher { +func (n *NetStore) getOrCreateFetcher(ctx context.Context, ref Address) *fetcher { if f := n.getFetcher(ref); f != nil { return f } @@ -176,7 +176,7 @@ func (n *NetStore) getOrCreateFetcher(ref Address) *fetcher { // no fetcher for the given address, we have to create a new one key := hex.EncodeToString(ref) // create the context during which fetching is kept alive - ctx, cancel := context.WithTimeout(context.Background(), fetcherTimeout) + cctx, cancel := context.WithTimeout(ctx, fetcherTimeout) // destroy is called when all requests finish destroy := func() { // remove fetcher from fetchers @@ -190,7 +190,7 @@ func (n *NetStore) getOrCreateFetcher(ref Address) *fetcher { // the peers which requested the chunk should not be requested to deliver it. peers := &sync.Map{} - fetcher := newFetcher(ref, n.NewNetFetcherFunc(ctx, ref, peers), destroy, peers, n.closeC) + fetcher := newFetcher(ref, n.NewNetFetcherFunc(cctx, ref, peers), destroy, peers, n.closeC) n.fetchers.Add(key, fetcher) return fetcher @@ -278,9 +278,9 @@ func (f *fetcher) Fetch(rctx context.Context) (Chunk, error) { if err := source.UnmarshalText([]byte(sourceIF.(string))); err != nil { return nil, err } - f.netFetcher.Offer(rctx, &source) + f.netFetcher.Offer(&source) } else { - f.netFetcher.Request(rctx, hopCount) + f.netFetcher.Request(hopCount) } // wait until either the chunk is delivered or the context is done diff --git a/swarm/storage/netstore_test.go b/swarm/storage/netstore_test.go index a6a9f551ab..88ec6c28f4 100644 --- a/swarm/storage/netstore_test.go +++ b/swarm/storage/netstore_test.go @@ -46,12 +46,12 @@ type mockNetFetcher struct { mu sync.Mutex } -func (m *mockNetFetcher) Offer(ctx context.Context, source *enode.ID) { +func (m *mockNetFetcher) Offer(source *enode.ID) { m.offerCalled = true m.sources = append(m.sources, source) } -func (m *mockNetFetcher) Request(ctx context.Context, hopCount uint8) { +func (m *mockNetFetcher) Request(hopCount uint8) { m.mu.Lock() defer m.mu.Unlock() From cde02e017ef2fb254f9b91888f4a14645c24890a Mon Sep 17 00:00:00 2001 From: gluk256 Date: Fri, 8 Feb 2019 20:05:10 +0400 Subject: [PATCH 09/22] swarm/pss: transition to whisper v6 (#19023) --- swarm/pss/client/client_test.go | 2 +- swarm/pss/forwarding_test.go | 2 +- swarm/pss/notify/notify_test.go | 2 +- swarm/pss/pss.go | 6 +++--- swarm/pss/pss_test.go | 2 +- swarm/pss/types.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/swarm/pss/client/client_test.go b/swarm/pss/client/client_test.go index 0d6788d679..1c6f2e522d 100644 --- a/swarm/pss/client/client_test.go +++ b/swarm/pss/client/client_test.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/state" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) type protoCtrl struct { diff --git a/swarm/pss/forwarding_test.go b/swarm/pss/forwarding_test.go index 2502977945..746d4dc404 100644 --- a/swarm/pss/forwarding_test.go +++ b/swarm/pss/forwarding_test.go @@ -12,7 +12,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pot" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) type testCase struct { diff --git a/swarm/pss/notify/notify_test.go b/swarm/pss/notify/notify_test.go index 95121b1d8f..cda069b9e1 100644 --- a/swarm/pss/notify/notify_test.go +++ b/swarm/pss/notify/notify_test.go @@ -19,7 +19,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pss" "github.com/ethereum/go-ethereum/swarm/state" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) var ( diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index ee942303c5..158ae4095d 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -38,7 +38,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/storage" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" "golang.org/x/crypto/sha3" ) @@ -686,7 +686,7 @@ func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, if err != nil { continue } - if !recvmsg.Validate() { + if !recvmsg.ValidateAndParse() { return nil, "", nil, fmt.Errorf("symmetrically encrypted message has invalid signature or is corrupt") } p.symKeyPoolMu.Lock() @@ -713,7 +713,7 @@ func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, return nil, "", nil, fmt.Errorf("could not decrypt message: %s", err) } // check signature (if signed), strip padding - if !recvmsg.Validate() { + if !recvmsg.ValidateAndParse() { return nil, "", nil, fmt.Errorf("invalid message") } pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src)) diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 0fb87be2c6..675b4cfcd6 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -50,7 +50,7 @@ import ( "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/pot" "github.com/ethereum/go-ethereum/swarm/state" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) var ( diff --git a/swarm/pss/types.go b/swarm/pss/types.go index ba963067cb..2ce1f5cfb0 100644 --- a/swarm/pss/types.go +++ b/swarm/pss/types.go @@ -26,7 +26,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/swarm/storage" - whisper "github.com/ethereum/go-ethereum/whisper/whisperv5" + whisper "github.com/ethereum/go-ethereum/whisper/whisperv6" ) const ( From 27e3f968194e2723279b60f71c79d4da9fc7577f Mon Sep 17 00:00:00 2001 From: Ferenc Szabo Date: Fri, 8 Feb 2019 17:07:11 +0100 Subject: [PATCH 10/22] swarm: CI race detector test adjustments (#19017) --- swarm/network/networkid_test.go | 4 +- swarm/network/simulations/overlay_test.go | 2 +- .../network/stream/snapshot_retrieval_test.go | 2 +- swarm/storage/common_test.go | 18 ++--- swarm/storage/ldbstore_test.go | 78 ++++++++----------- swarm/storage/memstore_test.go | 40 ++++------ 6 files changed, 60 insertions(+), 84 deletions(-) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index 8e68e5b521..9d47cf9f6a 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -44,7 +44,7 @@ var ( const ( NumberOfNets = 4 - MaxTimeout = 6 + MaxTimeout = 15 * time.Second ) func init() { @@ -146,7 +146,7 @@ func setupNetwork(numnodes int) (net *simulations.Network, err error) { return nil, fmt.Errorf("create node %d rpc client fail: %v", i, err) } //now setup and start event watching in order to know when we can upload - ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout*time.Second) + ctx, watchCancel := context.WithTimeout(context.Background(), MaxTimeout) defer watchCancel() watchSubscriptionEvents(ctx, nodes[i].ID(), client, errc, quitC) //on every iteration we connect to all previous ones diff --git a/swarm/network/simulations/overlay_test.go b/swarm/network/simulations/overlay_test.go index e56b646503..636a527591 100644 --- a/swarm/network/simulations/overlay_test.go +++ b/swarm/network/simulations/overlay_test.go @@ -32,7 +32,7 @@ import ( ) var ( - nodeCount = 16 + nodeCount = 10 ) //This test is used to test the overlay simulation. diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index f097e4180a..0f20f1be06 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -151,7 +151,7 @@ func runFileRetrievalTest(nodeCount int) error { return err } - ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancelSimRun := context.WithTimeout(context.Background(), 3*time.Minute) defer cancelSimRun() result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index 23d43beccf..6955ee8279 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -83,7 +83,7 @@ func newLDBStore(t *testing.T) (*LDBStore, func()) { return db, cleanup } -func mputRandomChunks(store ChunkStore, n int, chunksize int64) ([]Chunk, error) { +func mputRandomChunks(store ChunkStore, n int) ([]Chunk, error) { return mput(store, n, GenerateRandomChunk) } @@ -91,7 +91,7 @@ func mput(store ChunkStore, n int, f func(i int64) Chunk) (hs []Chunk, err error // put to localstore and wait for stored channel // does not check delivery error state errc := make(chan error) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() for i := int64(0); i < int64(n); i++ { chunk := f(ch.DefaultSize) @@ -159,8 +159,8 @@ func (r *brokenLimitedReader) Read(buf []byte) (int, error) { return r.lr.Read(buf) } -func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { - chunks, err := mputRandomChunks(m, n, chunksize) +func testStoreRandom(m ChunkStore, n int, t *testing.T) { + chunks, err := mputRandomChunks(m, n) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -170,8 +170,8 @@ func testStoreRandom(m ChunkStore, n int, chunksize int64, t *testing.T) { } } -func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { - chunks, err := mputRandomChunks(m, n, chunksize) +func testStoreCorrect(m ChunkStore, n int, t *testing.T) { + chunks, err := mputRandomChunks(m, n) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -195,7 +195,7 @@ func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { } } -func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { +func benchmarkStorePut(store ChunkStore, n int, b *testing.B) { chunks := make([]Chunk, n) i := 0 f := func(dataSize int64) Chunk { @@ -222,8 +222,8 @@ func benchmarkStorePut(store ChunkStore, n int, chunksize int64, b *testing.B) { } } -func benchmarkStoreGet(store ChunkStore, n int, chunksize int64, b *testing.B) { - chunks, err := mputRandomChunks(store, n, chunksize) +func benchmarkStoreGet(store ChunkStore, n int, b *testing.B) { + chunks, err := mputRandomChunks(store, n) if err != nil { b.Fatalf("expected no error, got %v", err) } diff --git a/swarm/storage/ldbstore_test.go b/swarm/storage/ldbstore_test.go index c87351323d..9e7aba545d 100644 --- a/swarm/storage/ldbstore_test.go +++ b/swarm/storage/ldbstore_test.go @@ -78,22 +78,22 @@ func testPoFunc(k Address) (ret uint8) { return uint8(Proximity(basekey, k[:])) } -func testDbStoreRandom(n int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreRandom(n int, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreRandom(db, n, chunksize, t) + testStoreRandom(db, n, t) } -func testDbStoreCorrect(n int, chunksize int64, mock bool, t *testing.T) { +func testDbStoreCorrect(n int, mock bool, t *testing.T) { db, cleanup, err := newTestDbStore(mock, false) defer cleanup() if err != nil { t.Fatalf("init dbStore failed: %v", err) } - testStoreCorrect(db, n, chunksize, t) + testStoreCorrect(db, n, t) } func TestMarkAccessed(t *testing.T) { @@ -137,35 +137,35 @@ func TestMarkAccessed(t *testing.T) { } func TestDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 0, false, t) + testDbStoreRandom(1, false, t) } func TestDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 4096, false, t) + testDbStoreCorrect(1, false, t) } func TestDbStoreRandom_1k(t *testing.T) { - testDbStoreRandom(1000, 0, false, t) + testDbStoreRandom(1000, false, t) } func TestDbStoreCorrect_1k(t *testing.T) { - testDbStoreCorrect(1000, 4096, false, t) + testDbStoreCorrect(1000, false, t) } func TestMockDbStoreRandom_1(t *testing.T) { - testDbStoreRandom(1, 0, true, t) + testDbStoreRandom(1, true, t) } func TestMockDbStoreCorrect_1(t *testing.T) { - testDbStoreCorrect(1, 4096, true, t) + testDbStoreCorrect(1, true, t) } func TestMockDbStoreRandom_1k(t *testing.T) { - testDbStoreRandom(1000, 0, true, t) + testDbStoreRandom(1000, true, t) } func TestMockDbStoreCorrect_1k(t *testing.T) { - testDbStoreCorrect(1000, 4096, true, t) + testDbStoreCorrect(1000, true, t) } func testDbStoreNotFound(t *testing.T, mock bool) { @@ -242,54 +242,38 @@ func TestMockIterator(t *testing.T) { testIterator(t, true) } -func benchmarkDbStorePut(n int, processors int, chunksize int64, mock bool, b *testing.B) { +func benchmarkDbStorePut(n int, mock bool, b *testing.B) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStorePut(db, n, chunksize, b) + benchmarkStorePut(db, n, b) } -func benchmarkDbStoreGet(n int, processors int, chunksize int64, mock bool, b *testing.B) { +func benchmarkDbStoreGet(n int, mock bool, b *testing.B) { db, cleanup, err := newTestDbStore(mock, true) defer cleanup() if err != nil { b.Fatalf("init dbStore failed: %v", err) } - benchmarkStoreGet(db, n, chunksize, b) + benchmarkStoreGet(db, n, b) } -func BenchmarkDbStorePut_1_500(b *testing.B) { - benchmarkDbStorePut(500, 1, 4096, false, b) +func BenchmarkDbStorePut_500(b *testing.B) { + benchmarkDbStorePut(500, false, b) } -func BenchmarkDbStorePut_8_500(b *testing.B) { - benchmarkDbStorePut(500, 8, 4096, false, b) +func BenchmarkDbStoreGet_500(b *testing.B) { + benchmarkDbStoreGet(500, false, b) } -func BenchmarkDbStoreGet_1_500(b *testing.B) { - benchmarkDbStoreGet(500, 1, 4096, false, b) +func BenchmarkMockDbStorePut_500(b *testing.B) { + benchmarkDbStorePut(500, true, b) } -func BenchmarkDbStoreGet_8_500(b *testing.B) { - benchmarkDbStoreGet(500, 8, 4096, false, b) -} - -func BenchmarkMockDbStorePut_1_500(b *testing.B) { - benchmarkDbStorePut(500, 1, 4096, true, b) -} - -func BenchmarkMockDbStorePut_8_500(b *testing.B) { - benchmarkDbStorePut(500, 8, 4096, true, b) -} - -func BenchmarkMockDbStoreGet_1_500(b *testing.B) { - benchmarkDbStoreGet(500, 1, 4096, true, b) -} - -func BenchmarkMockDbStoreGet_8_500(b *testing.B) { - benchmarkDbStoreGet(500, 8, 4096, true, b) +func BenchmarkMockDbStoreGet_500(b *testing.B) { + benchmarkDbStoreGet(500, true, b) } // TestLDBStoreWithoutCollectGarbage tests that we can put a number of random chunks in the LevelDB store, and @@ -302,7 +286,7 @@ func TestLDBStoreWithoutCollectGarbage(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err.Error()) } @@ -382,7 +366,7 @@ func testLDBStoreCollectGarbage(t *testing.T) { putCount = roundTarget } remaining -= putCount - chunks, err := mputRandomChunks(ldb, putCount, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, putCount) if err != nil { t.Fatal(err.Error()) } @@ -429,7 +413,7 @@ func TestLDBStoreAddRemove(t *testing.T) { defer cleanup() n := 100 - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatalf(err.Error()) } @@ -576,7 +560,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, int64(ch.DefaultSize)) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err.Error()) } @@ -589,7 +573,7 @@ func TestLDBStoreCollectGarbageAccessUnlikeIndex(t *testing.T) { t.Fatalf("fail add chunk #%d - %s: %v", i, chunks[i].Address(), err) } } - _, err = mputRandomChunks(ldb, 2, int64(ch.DefaultSize)) + _, err = mputRandomChunks(ldb, 2) if err != nil { t.Fatal(err.Error()) } @@ -621,7 +605,7 @@ func TestCleanIndex(t *testing.T) { ldb.setCapacity(uint64(capacity)) defer cleanup() - chunks, err := mputRandomChunks(ldb, n, 4096) + chunks, err := mputRandomChunks(ldb, n) if err != nil { t.Fatal(err) } @@ -752,7 +736,7 @@ func TestCleanIndex(t *testing.T) { } // check that the iterator quits properly - chunks, err = mputRandomChunks(ldb, 4100, 4096) + chunks, err = mputRandomChunks(ldb, 4100) if err != nil { t.Fatal(err) } diff --git a/swarm/storage/memstore_test.go b/swarm/storage/memstore_test.go index 6850d2d69a..8aaf486a7b 100644 --- a/swarm/storage/memstore_test.go +++ b/swarm/storage/memstore_test.go @@ -28,32 +28,32 @@ func newTestMemStore() *MemStore { return NewMemStore(storeparams, nil) } -func testMemStoreRandom(n int, chunksize int64, t *testing.T) { +func testMemStoreRandom(n int, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreRandom(m, n, chunksize, t) + testStoreRandom(m, n, t) } -func testMemStoreCorrect(n int, chunksize int64, t *testing.T) { +func testMemStoreCorrect(n int, t *testing.T) { m := newTestMemStore() defer m.Close() - testStoreCorrect(m, n, chunksize, t) + testStoreCorrect(m, n, t) } func TestMemStoreRandom_1(t *testing.T) { - testMemStoreRandom(1, 0, t) + testMemStoreRandom(1, t) } func TestMemStoreCorrect_1(t *testing.T) { - testMemStoreCorrect(1, 4104, t) + testMemStoreCorrect(1, t) } func TestMemStoreRandom_1k(t *testing.T) { - testMemStoreRandom(1000, 0, t) + testMemStoreRandom(1000, t) } func TestMemStoreCorrect_1k(t *testing.T) { - testMemStoreCorrect(100, 4096, t) + testMemStoreCorrect(100, t) } func TestMemStoreNotFound(t *testing.T) { @@ -66,32 +66,24 @@ func TestMemStoreNotFound(t *testing.T) { } } -func benchmarkMemStorePut(n int, processors int, chunksize int64, b *testing.B) { +func benchmarkMemStorePut(n int, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStorePut(m, n, chunksize, b) + benchmarkStorePut(m, n, b) } -func benchmarkMemStoreGet(n int, processors int, chunksize int64, b *testing.B) { +func benchmarkMemStoreGet(n int, b *testing.B) { m := newTestMemStore() defer m.Close() - benchmarkStoreGet(m, n, chunksize, b) + benchmarkStoreGet(m, n, b) } -func BenchmarkMemStorePut_1_500(b *testing.B) { - benchmarkMemStorePut(500, 1, 4096, b) +func BenchmarkMemStorePut_500(b *testing.B) { + benchmarkMemStorePut(500, b) } -func BenchmarkMemStorePut_8_500(b *testing.B) { - benchmarkMemStorePut(500, 8, 4096, b) -} - -func BenchmarkMemStoreGet_1_500(b *testing.B) { - benchmarkMemStoreGet(500, 1, 4096, b) -} - -func BenchmarkMemStoreGet_8_500(b *testing.B) { - benchmarkMemStoreGet(500, 8, 4096, b) +func BenchmarkMemStoreGet_500(b *testing.B) { + benchmarkMemStoreGet(500, b) } func TestMemStoreAndLDBStore(t *testing.T) { From 6cb7d52a29c68cdc4eafabb6dfe7594c288d151e Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 12 Feb 2019 08:34:08 +0100 Subject: [PATCH 11/22] swarm/docker: add global-store and split docker images (#19038) --- swarm/docker/Dockerfile | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/swarm/docker/Dockerfile b/swarm/docker/Dockerfile index 1ee4e97342..9450609dd8 100644 --- a/swarm/docker/Dockerfile +++ b/swarm/docker/Dockerfile @@ -10,14 +10,23 @@ RUN mkdir -p $GOPATH/src/github.com/ethereum && \ git checkout ${VERSION} && \ go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm && \ go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/swarm-smoke && \ - go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth && \ - cp $GOPATH/bin/swarm /swarm && cp $GOPATH/bin/geth /geth && cp $GOPATH/bin/swarm-smoke /swarm-smoke + go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/swarm/global-store && \ + go install -ldflags "-X main.gitCommit=${VERSION}" ./cmd/geth -# Release image with the required binaries and scripts -FROM alpine:3.8 +FROM alpine:3.8 as swarm-smoke WORKDIR / -COPY --from=builder /swarm /geth /swarm-smoke / -ADD run.sh /run.sh +COPY --from=builder /go/bin/swarm-smoke / ADD run-smoke.sh /run-smoke.sh +ENTRYPOINT ["/run-smoke.sh"] + +FROM alpine:3.8 as swarm-global-store +WORKDIR / +COPY --from=builder /go/bin/global-store / +ENTRYPOINT ["/global-store"] + +FROM alpine:3.8 as swarm +WORKDIR / +COPY --from=builder /go/bin/swarm /go/bin/geth / +ADD run.sh /run.sh ENTRYPOINT ["/run.sh"] From 3de19c8b31ab975eed1f7f276d31761f7f8b9af9 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 12 Feb 2019 10:55:25 +0100 Subject: [PATCH 12/22] build: use SFTP for launchpad uploads (#19037) * build: use sftp for launchpad uploads * .travis.yml: configure sftp export * build: update CI docs --- .travis.yml | 4 ++- build/ci-notes.md | 13 +++++++--- build/ci.go | 56 ++++++++++++++++++++++++++++------------- build/dput-launchpad.cf | 8 ++++++ 4 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 build/dput-launchpad.cf diff --git a/.travis.yml b/.travis.yml index f08083d3c3..c34e3ea8a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,8 +68,10 @@ matrix: - debhelper - dput - fakeroot + - python-bzrlib + - python-paramiko script: - - go run build/ci.go debsrc -signer "Go Ethereum Linux Builder " -upload ppa:ethereum/ethereum + - go run build/ci.go debsrc -upload ppa:ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - if: type = push diff --git a/build/ci-notes.md b/build/ci-notes.md index f5b0e869d6..ba27cdb1d7 100644 --- a/build/ci-notes.md +++ b/build/ci-notes.md @@ -7,11 +7,18 @@ Canonical. Packages of develop branch commits have suffix -unstable and cannot be installed alongside the stable version. Switching between release streams requires user intervention. +## Launchpad + The packages are built and served by launchpad.net. We generate a Debian source package for each distribution and upload it. Their builder picks up the source package, builds it and installs the new version into the PPA repository. Launchpad requires a valid signature -by a team member for source package uploads. The signing key is stored in an environment -variable which Travis CI makes available to certain builds. +by a team member for source package uploads. + +The signing key is stored in an environment variable which Travis CI makes available to +certain builds. Since Travis CI doesn't support FTP, SFTP is used to transfer the +packages. To set this up yourself, you need to create a Launchpad user and add a GPG key +and SSH key to it. Then encode both keys as base64 and configure 'secret' environment +variables `PPA_SIGNING_KEY` and `PPA_SSH_KEY` on Travis. We want to build go-ethereum with the most recent version of Go, irrespective of the Go version that is available in the main Ubuntu repository. In order to make this possible, @@ -27,7 +34,7 @@ Add the gophers PPA and install Go 1.10 and Debian packaging tools: $ sudo apt-add-repository ppa:gophers/ubuntu/archive $ sudo apt-get update - $ sudo apt-get install build-essential golang-1.10 devscripts debhelper + $ sudo apt-get install build-essential golang-1.10 devscripts debhelper python-bzrlib python-paramiko Create the source packages: diff --git a/build/ci.go b/build/ci.go index d2f4ce4fec..14d97135b4 100644 --- a/build/ci.go +++ b/build/ci.go @@ -441,11 +441,8 @@ func archiveBasename(arch string, archiveVersion string) string { func archiveUpload(archive string, blobstore string, signer string) error { // If signing was requested, generate the signature files if signer != "" { - pgpkey, err := base64.StdEncoding.DecodeString(os.Getenv(signer)) - if err != nil { - return fmt.Errorf("invalid base64 %s", signer) - } - if err := build.PGPSignFile(archive, archive+".asc", string(pgpkey)); err != nil { + key := getenvBase64(signer) + if err := build.PGPSignFile(archive, archive+".asc", string(key)); err != nil { return err } } @@ -489,6 +486,7 @@ func doDebianSource(cmdline []string) { var ( signer = flag.String("signer", "", `Signing key name, also used as package author`) upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`) + sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) now = time.Now() ) @@ -498,11 +496,7 @@ func doDebianSource(cmdline []string) { maybeSkipArchive(env) // Import the signing key. - if b64key := os.Getenv("PPA_SIGNING_KEY"); b64key != "" { - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatal("invalid base64 PPA_SIGNING_KEY") - } + if key := getenvBase64("PPA_SIGNING_KEY"); len(key) > 0 { gpg := exec.Command("gpg", "--import") gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) @@ -523,12 +517,45 @@ func doDebianSource(cmdline []string) { build.MustRunCommand("debsign", changes) } if *upload != "" { - build.MustRunCommand("dput", "--passive", "--no-upload-log", *upload, changes) + uploadDebianSource(*workdir, *upload, *sshUser, changes) } } } } +func uploadDebianSource(workdir, ppa, sshUser, changes string) { + // Create the dput config file. + dputConfig := filepath.Join(workdir, "dput.cf") + p := strings.Split(ppa, "/") + if len(p) != 2 { + log.Fatal("-upload PPA name must contain single /") + } + templateData := map[string]string{ + "LaunchpadUser": p[0], + "LaunchpadPPA": p[1], + "LaunchpadSSH": sshUser, + } + if sshkey := getenvBase64("PPA_SSH_KEY"); len(sshkey) > 0 { + idfile := filepath.Join(workdir, "sshkey") + ioutil.WriteFile(idfile, sshkey, 0600) + templateData["IdentityFile"] = idfile + } + build.Render("build/dput-launchpad.cf", dputConfig, 0644, templateData) + + // Run dput to do the upload. + dput := exec.Command("dput", "-c", dputConfig, "--no-upload-log", ppa, changes) + dput.Stdin = strings.NewReader("Yes\n") // accept SSH host key + build.MustRun(dput) +} + +func getenvBase64(variable string) []byte { + dec, err := base64.StdEncoding.DecodeString(os.Getenv(variable)) + if err != nil { + log.Fatal("invalid base64 " + variable) + } + return []byte(dec) +} + func makeWorkdir(wdflag string) string { var err error if wdflag != "" { @@ -800,15 +827,10 @@ func doAndroidArchive(cmdline []string) { os.Rename(archive, meta.Package+".aar") if *signer != "" && *deploy != "" { // Import the signing key into the local GPG instance - b64key := os.Getenv(*signer) - key, err := base64.StdEncoding.DecodeString(b64key) - if err != nil { - log.Fatalf("invalid base64 %s", *signer) - } + key := getenvBase64(*signer) gpg := exec.Command("gpg", "--import") gpg.Stdin = bytes.NewReader(key) build.MustRun(gpg) - keyID, err := build.PGPKeyID(string(key)) if err != nil { log.Fatal(err) diff --git a/build/dput-launchpad.cf b/build/dput-launchpad.cf new file mode 100644 index 0000000000..3063c3c07a --- /dev/null +++ b/build/dput-launchpad.cf @@ -0,0 +1,8 @@ +[{{.LaunchpadUser}}/{{.LaunchpadPPA}}] +fqdn = ppa.launchpad.net +method = sftp +incoming = ~{{.LaunchpadUser}}/ubuntu/{{.LaunchpadPPA}}/ +login = {{.LaunchpadSSH}} +{{ if .IdentityFile }} + ssh_options = IdentityFile {{.IdentityFile}} +{{ end }} From f48da43bae183a04a23d298cb1790d2f8d2cec51 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Feb 2019 11:29:05 +0100 Subject: [PATCH 13/22] common/fdlimit: cap on MacOS file limits, fixes #18994 (#19035) * common/fdlimit: cap on MacOS file limits, fixes #18994 * common/fdlimit: fix Maximum-check to respect OPEN_MAX * common/fdlimit: return error if OPEN_MAX is exceeded in Raise() * common/fdlimit: goimports * common/fdlimit: check value after setting fdlimit * common/fdlimit: make comment a bit more descriptive * cmd/utils: make fdlimit happy path a bit cleaner --- cmd/utils/flags.go | 5 +++-- common/fdlimit/fdlimit_freebsd.go | 11 +++++++---- common/fdlimit/fdlimit_test.go | 2 +- common/fdlimit/fdlimit_unix.go | 13 +++++++++---- common/fdlimit/fdlimit_windows.go | 4 ++-- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 66f533102e..22b7eb3d22 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -950,10 +950,11 @@ func makeDatabaseHandles() int { if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } - if err := fdlimit.Raise(uint64(limit)); err != nil { + raised, err := fdlimit.Raise(uint64(limit)) + if err != nil { Fatalf("Failed to raise file descriptor allowance: %v", err) } - return limit / 2 // Leave half for networking and other stuff + return int(raised / 2) // Leave half for networking and other stuff } // MakeAddress converts an account specified directly as a hex encoded string or diff --git a/common/fdlimit/fdlimit_freebsd.go b/common/fdlimit/fdlimit_freebsd.go index c126b0c265..5da4342379 100644 --- a/common/fdlimit/fdlimit_freebsd.go +++ b/common/fdlimit/fdlimit_freebsd.go @@ -26,11 +26,11 @@ import "syscall" // Raise tries to maximize the file descriptor allowance of this process // to the maximum hard-limit allowed by the OS. -func Raise(max uint64) error { +func Raise(max uint64) (uint64, error) { // Get the current limit var limit syscall.Rlimit if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { - return err + return 0, err } // Try to update the limit to the max allowance limit.Cur = limit.Max @@ -38,9 +38,12 @@ func Raise(max uint64) error { limit.Cur = int64(max) } if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { - return err + return 0, err } - return nil + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return limit.Cur, nil } // Current retrieves the number of file descriptors allowed to be opened by this diff --git a/common/fdlimit/fdlimit_test.go b/common/fdlimit/fdlimit_test.go index a9ee9ab36a..21362b8463 100644 --- a/common/fdlimit/fdlimit_test.go +++ b/common/fdlimit/fdlimit_test.go @@ -36,7 +36,7 @@ func TestFileDescriptorLimits(t *testing.T) { if limit, err := Current(); err != nil || limit <= 0 { t.Fatalf("failed to retrieve file descriptor limit (%d): %v", limit, err) } - if err := Raise(uint64(target)); err != nil { + if _, err := Raise(uint64(target)); err != nil { t.Fatalf("failed to raise file allowance") } if limit, err := Current(); err != nil || limit < target { diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go index a258132353..6701127515 100644 --- a/common/fdlimit/fdlimit_unix.go +++ b/common/fdlimit/fdlimit_unix.go @@ -22,11 +22,12 @@ import "syscall" // Raise tries to maximize the file descriptor allowance of this process // to the maximum hard-limit allowed by the OS. -func Raise(max uint64) error { +// Returns the size it was set to (may differ from the desired 'max') +func Raise(max uint64) (uint64, error) { // Get the current limit var limit syscall.Rlimit if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { - return err + return 0, err } // Try to update the limit to the max allowance limit.Cur = limit.Max @@ -34,9 +35,13 @@ func Raise(max uint64) error { limit.Cur = max } if err := syscall.Setrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { - return err + return 0, err } - return nil + // MacOS can silently apply further caps, so retrieve the actually set limit + if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil { + return 0, err + } + return limit.Cur, nil } // Current retrieves the number of file descriptors allowed to be opened by this diff --git a/common/fdlimit/fdlimit_windows.go b/common/fdlimit/fdlimit_windows.go index 863c58bedf..c9904cc8c2 100644 --- a/common/fdlimit/fdlimit_windows.go +++ b/common/fdlimit/fdlimit_windows.go @@ -20,7 +20,7 @@ import "errors" // Raise tries to maximize the file descriptor allowance of this process // to the maximum hard-limit allowed by the OS. -func Raise(max uint64) error { +func Raise(max uint64) (uint64, error) { // This method is NOP by design: // * Linux/Darwin counterparts need to manually increase per process limits // * On Windows Go uses the CreateFile API, which is limited to 16K files, non @@ -30,7 +30,7 @@ func Raise(max uint64) error { if max > 16384 { return errors.New("file descriptor limit (16384) reached") } - return nil + return max, nil } // Current retrieves the number of file descriptors allowed to be opened by this From edf976ee8e7e1561cf11cbdc5a0c5edb497dda34 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 12 Feb 2019 13:02:40 +0100 Subject: [PATCH 14/22] .travis.yml: fix upload destination (#19043) --- .travis.yml | 2 +- build/ci.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index c34e3ea8a9..9731ce181d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,7 +71,7 @@ matrix: - python-bzrlib - python-paramiko script: - - go run build/ci.go debsrc -upload ppa:ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - if: type = push diff --git a/build/ci.go b/build/ci.go index 14d97135b4..6ca08d9ef9 100644 --- a/build/ci.go +++ b/build/ci.go @@ -485,7 +485,7 @@ func maybeSkipArchive(env build.Environment) { func doDebianSource(cmdline []string) { var ( signer = flag.String("signer", "", `Signing key name, also used as package author`) - upload = flag.String("upload", "", `Where to upload the source package (usually "ppa:ethereum/ethereum")`) + upload = flag.String("upload", "", `Where to upload the source package (usually "ethereum/ethereum")`) sshUser = flag.String("sftp-user", "", `Username for SFTP upload (usually "geth-ci")`) workdir = flag.String("workdir", "", `Output directory for packages (uses temp dir if unset)`) now = time.Now() From 75d292bcf673e1b10ac883f8767d092d2f45fd9a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Feb 2019 14:00:02 +0100 Subject: [PATCH 15/22] clef: external signing fixes + signing data (#19003) * signer/clef: make use of json-rpc notification * signer: tidy up output of OnApprovedTx * accounts/external, signer: implement remote signing of text, make accounts_sign take hexdata * clef: added basic testscript * signer, external, api: add clique signing test to debug rpc, fix clique signing in clef * signer: fix clique interoperability between geth and clef * clef: rename networkid switch to chainid * clef: enable chainid flag * clef, signer: minor changes from review * clef: more tests for signer --- accounts/external/backend.go | 39 +++++++------ cmd/clef/intapi_changelog.md | 9 ++- cmd/clef/main.go | 14 ++++- cmd/clef/tests/testsigner.js | 73 ++++++++++++++++++++++++ internal/ethapi/api.go | 40 ++++++++++++++ internal/web3ext/web3ext.go | 6 ++ signer/core/api.go | 2 +- signer/core/cliui.go | 8 ++- signer/core/signed_data.go | 104 +++++++++++++++++++---------------- signer/core/stdioui.go | 18 ++++-- 10 files changed, 240 insertions(+), 73 deletions(-) create mode 100644 cmd/clef/tests/testsigner.js diff --git a/accounts/external/backend.go b/accounts/external/backend.go index 3b8d50f1b6..21a313b669 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" @@ -154,13 +153,31 @@ func (api *ExternalSigner) signHash(account accounts.Account, hash []byte) ([]by // SignData signs keccak256(data). The mimetype parameter describes the type of data being signed func (api *ExternalSigner) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { - // TODO! Replace this with a call to clef SignData with correct mime-type for Clique, once we - // have that in place - return api.signHash(account, crypto.Keccak256(data)) + var res hexutil.Bytes + var signAddress = common.NewMixedcaseAddress(account.Address) + if err := api.client.Call(&res, "account_signData", + mimeType, + &signAddress, // Need to use the pointer here, because of how MarshalJSON is defined + hexutil.Encode(data)); err != nil { + return nil, err + } + // If V is on 27/28-form, convert to to 0/1 for Clique + if mimeType == accounts.MimetypeClique && (res[64] == 27 || res[64] == 28) { + res[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use + } + return res, nil } func (api *ExternalSigner) SignText(account accounts.Account, text []byte) ([]byte, error) { - return api.signHash(account, accounts.TextHash(text)) + var res hexutil.Bytes + var signAddress = common.NewMixedcaseAddress(account.Address) + if err := api.client.Call(&res, "account_signData", + accounts.MimetypeTextPlain, + &signAddress, // Need to use the pointer here, because of how MarshalJSON is defined + hexutil.Encode(text)); err != nil { + return nil, err + } + return res, nil } func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { @@ -202,18 +219,6 @@ func (api *ExternalSigner) listAccounts() ([]common.Address, error) { return res, nil } -func (api *ExternalSigner) signCliqueBlock(a common.Address, rlpBlock hexutil.Bytes) (hexutil.Bytes, error) { - var sig hexutil.Bytes - if err := api.client.Call(&sig, "account_signData", core.ApplicationClique.Mime, a, rlpBlock); err != nil { - return nil, err - } - if sig[64] != 27 && sig[64] != 28 { - return nil, fmt.Errorf("invalid Ethereum signature (V is not 27 or 28)") - } - sig[64] -= 27 // Transform V from 27/28 to 0/1 for Clique use - return sig, nil -} - func (api *ExternalSigner) pingVersion() (string, error) { var v string if err := api.client.Call(&v, "account_version"); err != nil { diff --git a/cmd/clef/intapi_changelog.md b/cmd/clef/intapi_changelog.md index 6388af7303..205aa1a27b 100644 --- a/cmd/clef/intapi_changelog.md +++ b/cmd/clef/intapi_changelog.md @@ -1,8 +1,15 @@ ### Changelog for internal API (ui-api) +### 3.2.0 + +* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification): + +> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request. +> +> Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error" ### 3.1.0 -* Add `ContentType string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations. +* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest EIP-191 and EIP-712 implementations. ### 3.0.0 diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 38821102dc..279b28d757 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -44,6 +44,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/signer/core" @@ -83,6 +84,11 @@ var ( Value: DefaultConfigDir(), Usage: "Directory for Clef configuration", } + chainIdFlag = cli.Int64Flag{ + Name: "chainid", + Value: params.MainnetChainConfig.ChainID.Int64(), + Usage: "Chain id to use for signing (1=mainnet, 3=ropsten, 4=rinkeby, 5=Goerli)", + } rpcPortFlag = cli.IntFlag{ Name: "rpcport", Usage: "HTTP-RPC server listening port", @@ -178,7 +184,7 @@ func init() { logLevelFlag, keystoreFlag, configdirFlag, - utils.NetworkIdFlag, + chainIdFlag, utils.LightKDFFlag, utils.NoUSBFlag, utils.RPCListenAddrFlag, @@ -402,9 +408,13 @@ func signer(c *cli.Context) error { } } } + log.Info("Starting signer", "chainid", c.GlobalInt64(chainIdFlag.Name), + "keystore", c.GlobalString(keystoreFlag.Name), + "light-kdf", c.GlobalBool(utils.LightKDFFlag.Name), + "advanced", c.GlobalBool(advancedMode.Name)) apiImpl := core.NewSignerAPI( - c.GlobalInt64(utils.NetworkIdFlag.Name), + c.GlobalInt64(chainIdFlag.Name), c.GlobalString(keystoreFlag.Name), c.GlobalBool(utils.NoUSBFlag.Name), ui, db, diff --git a/cmd/clef/tests/testsigner.js b/cmd/clef/tests/testsigner.js new file mode 100644 index 0000000000..86b2c45395 --- /dev/null +++ b/cmd/clef/tests/testsigner.js @@ -0,0 +1,73 @@ +// This file is a test-utility for testing clef-functionality +// +// Start clef with +// +// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc +// +// Start geth with +// +// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js +// +// and in the console simply invoke +// +// > test() +// +// You can reload the file via `reload()` + +function reload(){ + loadScript("./cmd/clef/tests/testsigner.js"); +} + +function init(){ + if (typeof accts == 'undefined' || accts.length == 0){ + accts = eth.accounts + console.log("Got accounts ", accts); + } +} +init() +function testTx(){ + if( accts && accts.length > 0) { + var a = accts[0] + var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1}) + var v = parseInt(txdata.tx.v) + console.log("V value: ", v) + if (v == 37 || v == 38){ + console.log("Mainnet 155-protected chainid was used") + } + if (v == 27 || v == 28){ + throw new Error("Mainnet chainid was used, but without replay protection!") + } + } +} +function testSignText(){ + if( accts && accts.length > 0){ + var a = accts[0] + var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world + console.log("signing response", r) + } +} +function testClique(){ + if( accts && accts.length > 0){ + var a = accts[0] + var r = debug.testSignCliqueBlock(a, 0); // Sign genesis + console.log("signing response", r) + if( a != r){ + throw new Error("Requested signing by "+a+ " but got sealer "+r) + } + } +} + +function test(){ + var tests = [ + testTx, + testSignText, + testClique, + ] + for( i in tests){ + try{ + tests[i]() + }catch(err){ + console.log(err) + } + } + } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index b3fd4522a9..4d5ef27da1 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -1469,6 +1470,45 @@ func (api *PublicDebugAPI) GetBlockRlp(ctx context.Context, number uint64) (stri return fmt.Sprintf("%x", encoded), nil } +// TestSignCliqueBlock fetches the given block number, and attempts to sign it as a clique header with the +// given address, returning the address of the recovered signature +// +// This is a temporary method to debug the externalsigner integration, +// TODO: Remove this method when the integration is mature +func (api *PublicDebugAPI) TestSignCliqueBlock(ctx context.Context, address common.Address, number uint64) (common.Address, error) { + block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) + if block == nil { + return common.Address{}, fmt.Errorf("block #%d not found", number) + } + header := block.Header() + header.Extra = make([]byte, 32+65) + encoded := clique.CliqueRLP(header) + + // Look up the wallet containing the requested signer + account := accounts.Account{Address: address} + wallet, err := api.b.AccountManager().Find(account) + if err != nil { + return common.Address{}, err + } + + signature, err := wallet.SignData(account, accounts.MimetypeClique, encoded) + if err != nil { + return common.Address{}, err + } + sealHash := clique.SealHash(header).Bytes() + log.Info("test signing of clique block", + "Sealhash", fmt.Sprintf("%x", sealHash), + "signature", fmt.Sprintf("%x", signature)) + pubkey, err := crypto.Ecrecover(sealHash, signature) + if err != nil { + return common.Address{}, err + } + var signer common.Address + copy(signer[:], crypto.Keccak256(pubkey[1:])[12:]) + + return signer, nil +} + // PrintBlock retrieves a block and returns its pretty printed form. func (api *PublicDebugAPI) PrintBlock(ctx context.Context, number uint64) (string, error) { block, _ := api.b.BlockByNumber(ctx, rpc.BlockNumber(number)) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 6b98c8b7e1..df69d0b94a 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -231,6 +231,12 @@ web3._extend({ call: 'debug_getBlockRlp', params: 1 }), + new web3._extend.Method({ + name: 'testSignCliqueBlock', + call: 'debug_testSignCliqueBlock', + params: 2, + inputFormatters: [web3._extend.formatters.inputAddressFormatter, null], + }), new web3._extend.Method({ name: 'setHead', call: 'debug_setHead', diff --git a/signer/core/api.go b/signer/core/api.go index 0521dce47b..25057dd128 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -41,7 +41,7 @@ const ( // ExternalAPIVersion -- see extapi_changelog.md ExternalAPIVersion = "5.0.0" // InternalAPIVersion -- see intapi_changelog.md - InternalAPIVersion = "3.1.0" + InternalAPIVersion = "3.2.0" ) // ExternalAPI defines the external API through which signing requests are made. diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 71d489d459..3ffc6b1916 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -18,12 +18,12 @@ package core import ( "bufio" + "encoding/json" "fmt" "os" "strings" "sync" - "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" @@ -264,7 +264,11 @@ func (ui *CommandlineUI) ShowInfo(message string) { func (ui *CommandlineUI) OnApprovedTx(tx ethapi.SignTransactionResult) { fmt.Printf("Transaction signed:\n ") - spew.Dump(tx.Tx) + if jsn, err := json.MarshalIndent(tx.Tx, " ", " "); err != nil { + fmt.Printf("WARN: marshalling error %v\n", err) + } else { + fmt.Println(string(jsn)) + } } func (ui *CommandlineUI) OnSignerStartup(info StartupInfo) { diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index ac0b97bcad..e481f5e1d4 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -121,8 +121,8 @@ var typedDataReferenceTypeRegexp = regexp.MustCompile(`^[A-Z](\w*)(\[\])?$`) // sign receives a request and produces a signature // Note, the produced signature conforms to the secp256k1 curve R, S and V values, -// where the V value will be 27 or 28 for legacy reasons. -func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) (hexutil.Bytes, error) { +// where the V value will be 27 or 28 for legacy reasons, if legacyV==true. +func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest, legacyV bool) (hexutil.Bytes, error) { // We make the request prior to looking up if we actually have the account, to prevent // account-enumeration via the API @@ -140,11 +140,13 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) ( return nil, err } // Sign the data with the wallet - signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Hash) + signature, err := wallet.SignDataWithPassphrase(account, res.Password, req.ContentType, req.Rawdata) if err != nil { return nil, err } - signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper + if legacyV { + signature[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper + } return signature, nil } @@ -153,17 +155,16 @@ func (api *SignerAPI) sign(addr common.MixedcaseAddress, req *SignDataRequest) ( // // Different types of validation occur. func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (hexutil.Bytes, error) { - var req, err = api.determineSignatureFormat(ctx, contentType, addr, data) + var req, transformV, err = api.determineSignatureFormat(ctx, contentType, addr, data) if err != nil { return nil, err } - signature, err := api.sign(addr, req) + signature, err := api.sign(addr, req, transformV) if err != nil { api.UI.ShowError(err.Error()) return nil, err } - return signature, nil } @@ -173,12 +174,14 @@ func (api *SignerAPI) SignData(ctx context.Context, contentType string, addr com // charset, ok := params["charset"] // As it is now, we accept any charset and just treat it as 'raw'. // This method returns the mimetype for signing along with the request -func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, error) { - var req *SignDataRequest - +func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType string, addr common.MixedcaseAddress, data interface{}) (*SignDataRequest, bool, error) { + var ( + req *SignDataRequest + useEthereumV = true // Default to use V = 27 or 28, the legacy Ethereum format + ) mediaType, _, err := mime.ParseMediaType(contentType) if err != nil { - return nil, err + return nil, useEthereumV, err } switch mediaType { @@ -186,7 +189,7 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType // Data with an intended validator validatorData, err := UnmarshalValidatorData(data) if err != nil { - return nil, err + return nil, useEthereumV, err } sighash, msg := SignTextValidator(validatorData) message := []*NameValueType{ @@ -201,55 +204,63 @@ func (api *SignerAPI) determineSignatureFormat(ctx context.Context, contentType // Clique is the Ethereum PoA standard stringData, ok := data.(string) if !ok { - return nil, fmt.Errorf("input for %v plain must be an hex-encoded string", ApplicationClique.Mime) + return nil, useEthereumV, fmt.Errorf("input for %v must be an hex-encoded string", ApplicationClique.Mime) } cliqueData, err := hexutil.Decode(stringData) if err != nil { - return nil, err + return nil, useEthereumV, err } header := &types.Header{} if err := rlp.DecodeBytes(cliqueData, header); err != nil { - return nil, err + return nil, useEthereumV, err + } + // The incoming clique header is already truncated, sent to us with a extradata already shortened + if len(header.Extra) < 65 { + // Need to add it back, to get a suitable length for hashing + newExtra := make([]byte, len(header.Extra)+65) + copy(newExtra, header.Extra) + header.Extra = newExtra } // Get back the rlp data, encoded by us - cliqueData = clique.CliqueRLP(header) - sighash, err := SignCliqueHeader(header) + sighash, cliqueRlp, err := cliqueHeaderHashAndRlp(header) if err != nil { - return nil, err + return nil, useEthereumV, err } message := []*NameValueType{ { - Name: "Clique block", + Name: "Clique header", Typ: "clique", - Value: fmt.Sprintf("clique block %d [0x%x]", header.Number, header.Hash()), + Value: fmt.Sprintf("clique header %d [0x%x]", header.Number, header.Hash()), }, } - req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueData, Message: message, Hash: sighash} + // Clique uses V on the form 0 or 1 + useEthereumV = false + req = &SignDataRequest{ContentType: mediaType, Rawdata: cliqueRlp, Message: message, Hash: sighash} default: // also case TextPlain.Mime: // Calculates an Ethereum ECDSA signature for: // hash = keccak256("\x19${byteVersion}Ethereum Signed Message:\n${message length}${message}") // We expect it to be a string - stringData, ok := data.(string) - if !ok { - return nil, fmt.Errorf("input for text/plain must be a string") + if stringData, ok := data.(string); !ok { + return nil, useEthereumV, fmt.Errorf("input for text/plain must be an hex-encoded string") + } else { + if textData, err := hexutil.Decode(stringData); err != nil { + return nil, useEthereumV, err + } else { + sighash, msg := accounts.TextAndHash(textData) + message := []*NameValueType{ + { + Name: "message", + Typ: "text/plain", + Value: msg, + }, + } + req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash} + } } - //plainData, err := hexutil.Decode(stringdata) - //if err != nil { - // return nil, err - //} - sighash, msg := accounts.TextAndHash([]byte(stringData)) - message := []*NameValueType{ - { - Name: "message", - Typ: "text/plain", - Value: msg, - }, - } - req = &SignDataRequest{ContentType: mediaType, Rawdata: []byte(msg), Message: message, Hash: sighash} } req.Address = addr req.Meta = MetadataFromContext(ctx) - return req, nil + return req, useEthereumV, nil } @@ -262,20 +273,21 @@ func SignTextValidator(validatorData ValidatorData) (hexutil.Bytes, string) { return crypto.Keccak256([]byte(msg)), msg } -// SignCliqueHeader returns the hash which is used as input for the proof-of-authority +// cliqueHeaderHashAndRlp returns the hash which is used as input for the proof-of-authority // signing. It is the hash of the entire header apart from the 65 byte signature // contained at the end of the extra data. // // The method requires the extra data to be at least 65 bytes -- the original implementation // in clique.go panics if this is the case, thus it's been reimplemented here to avoid the panic // and simply return an error instead -func SignCliqueHeader(header *types.Header) (hexutil.Bytes, error) { - //hash := common.Hash{} +func cliqueHeaderHashAndRlp(header *types.Header) (hash, rlp []byte, err error) { if len(header.Extra) < 65 { - return nil, fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra)) + err = fmt.Errorf("clique header extradata too short, %d < 65", len(header.Extra)) + return } - hash := clique.SealHash(header) - return hash.Bytes(), nil + rlp = clique.CliqueRLP(header) + hash = clique.SealHash(header).Bytes() + return hash, rlp, err } // SignTypedData signs EIP-712 conformant typed data @@ -293,7 +305,7 @@ func (api *SignerAPI) SignTypedData(ctx context.Context, addr common.MixedcaseAd sighash := crypto.Keccak256(rawData) message := typedData.Format() req := &SignDataRequest{ContentType: DataTyped.Mime, Rawdata: rawData, Message: message, Hash: sighash} - signature, err := api.sign(addr, req) + signature, err := api.sign(addr, req, true) if err != nil { api.UI.ShowError(err.Error()) return nil, err @@ -722,7 +734,7 @@ func (nvt *NameValueType) Pprint(depth int) string { output.WriteString(sublevel) } } else { - output.WriteString(fmt.Sprintf("%s\n", nvt.Value)) + output.WriteString(fmt.Sprintf("%q\n", nvt.Value)) } return output.String() } diff --git a/signer/core/stdioui.go b/signer/core/stdioui.go index 64032386fc..e169a8bc08 100644 --- a/signer/core/stdioui.go +++ b/signer/core/stdioui.go @@ -49,6 +49,16 @@ func (ui *StdIOUI) dispatch(serviceMethod string, args interface{}, reply interf return err } +// notify sends a request over the stdio, and does not listen for a response +func (ui *StdIOUI) notify(serviceMethod string, args interface{}) error { + ctx := context.Background() + err := ui.client.Notify(ctx, serviceMethod, args) + if err != nil { + log.Info("Error", "exc", err.Error()) + } + return err +} + func (ui *StdIOUI) ApproveTx(request *SignTxRequest) (SignTxResponse, error) { var result SignTxResponse err := ui.dispatch("ApproveTx", request, &result) @@ -86,27 +96,27 @@ func (ui *StdIOUI) ApproveNewAccount(request *NewAccountRequest) (NewAccountResp } func (ui *StdIOUI) ShowError(message string) { - err := ui.dispatch("ShowError", &Message{message}, nil) + err := ui.notify("ShowError", &Message{message}) if err != nil { log.Info("Error calling 'ShowError'", "exc", err.Error(), "msg", message) } } func (ui *StdIOUI) ShowInfo(message string) { - err := ui.dispatch("ShowInfo", Message{message}, nil) + err := ui.notify("ShowInfo", Message{message}) if err != nil { log.Info("Error calling 'ShowInfo'", "exc", err.Error(), "msg", message) } } func (ui *StdIOUI) OnApprovedTx(tx ethapi.SignTransactionResult) { - err := ui.dispatch("OnApprovedTx", tx, nil) + err := ui.notify("OnApprovedTx", tx) if err != nil { log.Info("Error calling 'OnApprovedTx'", "exc", err.Error(), "tx", tx) } } func (ui *StdIOUI) OnSignerStartup(info StartupInfo) { - err := ui.dispatch("OnSignerStartup", info, nil) + err := ui.notify("OnSignerStartup", info) if err != nil { log.Info("Error calling 'OnSignerStartup'", "exc", err.Error(), "info", info) } From b5d471a73905247406dcbe5ffac6087f80896374 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Feb 2019 17:38:46 +0100 Subject: [PATCH 16/22] clef: bidirectional communication with UI (#19018) * clef: initial implementation of bidirectional RPC communication for the UI * signer: fix tests to pass + formatting * clef: fix unused import + formatting * signer: gosimple nitpicks --- cmd/clef/extapi_changelog.md | 5 + cmd/clef/intapi_changelog.md | 24 +++++ cmd/clef/main.go | 33 +++--- signer/core/api.go | 163 ++++++++++------------------ signer/core/api_test.go | 22 ++-- signer/core/auditlog.go | 13 +-- signer/core/cliui.go | 5 +- signer/core/stdioui.go | 8 +- signer/core/types.go | 25 ----- signer/core/uiapi.go | 201 +++++++++++++++++++++++++++++++++++ signer/rules/rules.go | 9 +- signer/rules/rules_test.go | 12 ++- 12 files changed, 338 insertions(+), 182 deletions(-) create mode 100644 signer/core/uiapi.go diff --git a/cmd/clef/extapi_changelog.md b/cmd/clef/extapi_changelog.md index 25f819bddb..dc7e65c01b 100644 --- a/cmd/clef/extapi_changelog.md +++ b/cmd/clef/extapi_changelog.md @@ -1,5 +1,10 @@ ### Changelog for external API +### 6.0.0 + +* `New` was changed to deliver only an address, not the full `Account` data +* `Export` was moved from External API to the UI Server API + #### 5.0.0 * The external `account_EcRecover`-method was reimplemented. diff --git a/cmd/clef/intapi_changelog.md b/cmd/clef/intapi_changelog.md index 205aa1a27b..db3353bb7d 100644 --- a/cmd/clef/intapi_changelog.md +++ b/cmd/clef/intapi_changelog.md @@ -1,5 +1,29 @@ ### Changelog for internal API (ui-api) +### 4.0.0 + +* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are: + - `clef_listWallets` + - `clef_listAccounts` + - `clef_listWallets` + - `clef_deriveAccount` + - `clef_importRawKey` + - `clef_openWallet` + - `clef_chainId` + - `clef_setChainId` + - `clef_export` + - `clef_import` + +* The type `Account` was modified (the json-field `type` was removed), to consist of + +```golang +type Account struct { + Address common.Address `json:"address"` // Ethereum account address derived from the key + URL URL `json:"url"` // Optional resource locator within a backend +} +``` + + ### 3.2.0 * Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification): diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 279b28d757..e1a44835a3 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -344,7 +344,7 @@ func signer(c *cli.Context) error { return err } var ( - ui core.SignerUI + ui core.UIClientAPI ) if c.GlobalBool(stdiouiFlag.Name) { log.Info("Using stdin/stdout as UI-channel") @@ -408,18 +408,21 @@ func signer(c *cli.Context) error { } } } - log.Info("Starting signer", "chainid", c.GlobalInt64(chainIdFlag.Name), - "keystore", c.GlobalString(keystoreFlag.Name), - "light-kdf", c.GlobalBool(utils.LightKDFFlag.Name), - "advanced", c.GlobalBool(advancedMode.Name)) + var ( + chainId = c.GlobalInt64(chainIdFlag.Name) + ksLoc = c.GlobalString(keystoreFlag.Name) + lightKdf = c.GlobalBool(utils.LightKDFFlag.Name) + advanced = c.GlobalBool(advancedMode.Name) + nousb = c.GlobalBool(utils.NoUSBFlag.Name) + ) + log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc, + "light-kdf", lightKdf, "advanced", advanced) + am := core.StartClefAccountManager(ksLoc, nousb, lightKdf) + apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced) - apiImpl := core.NewSignerAPI( - c.GlobalInt64(chainIdFlag.Name), - c.GlobalString(keystoreFlag.Name), - c.GlobalBool(utils.NoUSBFlag.Name), - ui, db, - c.GlobalBool(utils.LightKDFFlag.Name), - c.GlobalBool(advancedMode.Name)) + // Establish the bidirectional communication, by creating a new UI backend and registering + // it with the UI. + ui.RegisterUIServer(core.NewUIServerAPI(apiImpl)) api = apiImpl // Audit logging if logfile := c.GlobalString(auditLogFlag.Name); logfile != "" { @@ -539,7 +542,7 @@ func homeDir() string { } return "" } -func readMasterKey(ctx *cli.Context, ui core.SignerUI) ([]byte, error) { +func readMasterKey(ctx *cli.Context, ui core.UIClientAPI) ([]byte, error) { var ( file string configDir = ctx.GlobalString(configdirFlag.Name) @@ -674,10 +677,6 @@ func testExternalUI(api *core.SignerAPI) { checkErr("List", err) _, err = api.New(ctx) checkErr("New", err) - _, err = api.Export(ctx, common.Address{}) - checkErr("Export", err) - _, err = api.Import(ctx, json.RawMessage{}) - checkErr("Import", err) api.UI.ShowInfo("Tests completed") diff --git a/signer/core/api.go b/signer/core/api.go index 25057dd128..30888f8429 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "math/big" "reflect" @@ -39,9 +38,9 @@ const ( // numberOfAccountsToDerive For hardware wallets, the number of accounts to derive numberOfAccountsToDerive = 10 // ExternalAPIVersion -- see extapi_changelog.md - ExternalAPIVersion = "5.0.0" + ExternalAPIVersion = "6.0.0" // InternalAPIVersion -- see intapi_changelog.md - InternalAPIVersion = "3.2.0" + InternalAPIVersion = "4.0.0" ) // ExternalAPI defines the external API through which signing requests are made. @@ -49,7 +48,7 @@ type ExternalAPI interface { // List available accounts List(ctx context.Context) ([]common.Address, error) // New request to create a new account - New(ctx context.Context) (accounts.Account, error) + New(ctx context.Context) (common.Address, error) // SignTransaction request to sign the specified transaction SignTransaction(ctx context.Context, args SendTxArgs, methodSelector *string) (*ethapi.SignTransactionResult, error) // SignData - request to sign the given data (plus prefix) @@ -58,17 +57,13 @@ type ExternalAPI interface { SignTypedData(ctx context.Context, addr common.MixedcaseAddress, data TypedData) (hexutil.Bytes, error) // EcRecover - recover public key from given message and signature EcRecover(ctx context.Context, data hexutil.Bytes, sig hexutil.Bytes) (common.Address, error) - // Export - request to export an account - Export(ctx context.Context, addr common.Address) (json.RawMessage, error) - // Import - request to import an account - // Should be moved to Internal API, in next phase when we have - // bi-directional communication - //Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) + // Version info about the APIs Version(ctx context.Context) (string, error) } -// SignerUI specifies what method a UI needs to implement to be able to be used as a UI for the signer -type SignerUI interface { +// UIClientAPI specifies what method a UI needs to implement to be able to be used as a +// UI for the signer +type UIClientAPI interface { // ApproveTx prompt the user for confirmation to request to sign Transaction ApproveTx(request *SignTxRequest) (SignTxResponse, error) // ApproveSignData prompt the user for confirmation to request to sign data @@ -95,13 +90,15 @@ type SignerUI interface { // OnInputRequired is invoked when clef requires user input, for example master password or // pin-code for unlocking hardware wallets OnInputRequired(info UserInputRequest) (UserInputResponse, error) + // RegisterUIServer tells the UI to use the given UIServerAPI for ui->clef communication + RegisterUIServer(api *UIServerAPI) } // SignerAPI defines the actual implementation of ExternalAPI type SignerAPI struct { chainID *big.Int am *accounts.Manager - UI SignerUI + UI UIClientAPI validator *Validator rejectMode bool } @@ -115,6 +112,37 @@ type Metadata struct { Origin string `json:"Origin"` } +func StartClefAccountManager(ksLocation string, nousb, lightKDF bool) *accounts.Manager { + var ( + backends []accounts.Backend + n, p = keystore.StandardScryptN, keystore.StandardScryptP + ) + if lightKDF { + n, p = keystore.LightScryptN, keystore.LightScryptP + } + // support password based accounts + if len(ksLocation) > 0 { + backends = append(backends, keystore.NewKeyStore(ksLocation, n, p)) + } + if !nousb { + // Start a USB hub for Ledger hardware wallets + if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { + log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) + } else { + backends = append(backends, ledgerhub) + log.Debug("Ledger support enabled") + } + // Start a USB hub for Trezor hardware wallets + if trezorhub, err := usbwallet.NewTrezorHub(); err != nil { + log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err)) + } else { + backends = append(backends, trezorhub) + log.Debug("Trezor support enabled") + } + } + return accounts.NewManager(backends...) +} + // MetadataFromContext extracts Metadata from a given context.Context func MetadataFromContext(ctx context.Context) Metadata { m := Metadata{"NA", "NA", "NA", "", ""} // batman @@ -199,11 +227,11 @@ type ( Password string `json:"password"` } ListRequest struct { - Accounts []Account `json:"accounts"` - Meta Metadata `json:"meta"` + Accounts []accounts.Account `json:"accounts"` + Meta Metadata `json:"meta"` } ListResponse struct { - Accounts []Account `json:"accounts"` + Accounts []accounts.Account `json:"accounts"` } Message struct { Text string `json:"text"` @@ -234,38 +262,11 @@ var ErrRequestDenied = errors.New("Request denied") // key that is generated when a new Account is created. // noUSB disables USB support that is required to support hardware devices such as // ledger and trezor. -func NewSignerAPI(chainID int64, ksLocation string, noUSB bool, ui SignerUI, abidb *AbiDb, lightKDF bool, advancedMode bool) *SignerAPI { - var ( - backends []accounts.Backend - n, p = keystore.StandardScryptN, keystore.StandardScryptP - ) - if lightKDF { - n, p = keystore.LightScryptN, keystore.LightScryptP - } - // support password based accounts - if len(ksLocation) > 0 { - backends = append(backends, keystore.NewKeyStore(ksLocation, n, p)) - } +func NewSignerAPI(am *accounts.Manager, chainID int64, noUSB bool, ui UIClientAPI, abidb *AbiDb, advancedMode bool) *SignerAPI { if advancedMode { log.Info("Clef is in advanced mode: will warn instead of reject") } - if !noUSB { - // Start a USB hub for Ledger hardware wallets - if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { - log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) - } else { - backends = append(backends, ledgerhub) - log.Debug("Ledger support enabled") - } - // Start a USB hub for Trezor hardware wallets - if trezorhub, err := usbwallet.NewTrezorHub(); err != nil { - log.Warn(fmt.Sprintf("Failed to start Trezor hub, disabling: %v", err)) - } else { - backends = append(backends, trezorhub) - log.Debug("Trezor support enabled") - } - } - signer := &SignerAPI{big.NewInt(chainID), accounts.NewManager(backends...), ui, NewValidator(abidb), !advancedMode} + signer := &SignerAPI{big.NewInt(chainID), am, ui, NewValidator(abidb), !advancedMode} if !noUSB { signer.startUSBListener() } @@ -358,12 +359,9 @@ func (api *SignerAPI) startUSBListener() { // List returns the set of wallet this signer manages. Each wallet can contain // multiple accounts. func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) { - var accs []Account + var accs []accounts.Account for _, wallet := range api.am.Wallets() { - for _, acc := range wallet.Accounts() { - acc := Account{Typ: "Account", URL: wallet.URL(), Address: acc.Address} - accs = append(accs, acc) - } + accs = append(accs, wallet.Accounts()...) } result, err := api.UI.ApproveListing(&ListRequest{Accounts: accs, Meta: MetadataFromContext(ctx)}) if err != nil { @@ -373,7 +371,6 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) { return nil, ErrRequestDenied } - addresses := make([]common.Address, 0) for _, acc := range result.Accounts { addresses = append(addresses, acc.Address) @@ -385,10 +382,10 @@ func (api *SignerAPI) List(ctx context.Context) ([]common.Address, error) { // New creates a new password protected Account. The private key is protected with // the given password. Users are responsible to backup the private key that is stored // in the keystore location thas was specified when this API was created. -func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) { +func (api *SignerAPI) New(ctx context.Context) (common.Address, error) { be := api.am.Backends(keystore.KeyStoreType) if len(be) == 0 { - return accounts.Account{}, errors.New("password based accounts not supported") + return common.Address{}, errors.New("password based accounts not supported") } var ( resp NewAccountResponse @@ -398,20 +395,21 @@ func (api *SignerAPI) New(ctx context.Context) (accounts.Account, error) { for i := 0; i < 3; i++ { resp, err = api.UI.ApproveNewAccount(&NewAccountRequest{MetadataFromContext(ctx)}) if err != nil { - return accounts.Account{}, err + return common.Address{}, err } if !resp.Approved { - return accounts.Account{}, ErrRequestDenied + return common.Address{}, ErrRequestDenied } if pwErr := ValidatePasswordFormat(resp.Password); pwErr != nil { api.UI.ShowError(fmt.Sprintf("Account creation attempt #%d failed due to password requirements: %v", (i + 1), pwErr)) } else { // No error - return be[0].(*keystore.KeyStore).NewAccount(resp.Password) + acc, err := be[0].(*keystore.KeyStore).NewAccount(resp.Password) + return acc.Address, err } } // Otherwise fail, with generic error message - return accounts.Account{}, errors.New("account creation failed") + return common.Address{}, errors.New("account creation failed") } // logDiff logs the difference between the incoming (original) transaction and the one returned from the signer. @@ -521,57 +519,6 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args SendTxArgs, meth } -// Export returns encrypted private key associated with the given address in web3 keystore format. -func (api *SignerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) { - res, err := api.UI.ApproveExport(&ExportRequest{Address: addr, Meta: MetadataFromContext(ctx)}) - - if err != nil { - return nil, err - } - if !res.Approved { - return nil, ErrRequestDenied - } - // Look up the wallet containing the requested signer - wallet, err := api.am.Find(accounts.Account{Address: addr}) - if err != nil { - return nil, err - } - if wallet.URL().Scheme != keystore.KeyStoreScheme { - return nil, fmt.Errorf("Account is not a keystore-account") - } - return ioutil.ReadFile(wallet.URL().Path) -} - -// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be -// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful -// decryption it will encrypt the key with the given newPassphrase and store it in the keystore. -// OBS! This method is removed from the public API. It should not be exposed on the external API -// for a couple of reasons: -// 1. Even though it is encrypted, it should still be seen as sensitive data -// 2. It can be used to DoS clef, by using malicious data with e.g. extreme large -// values for the kdfparams. -func (api *SignerAPI) Import(ctx context.Context, keyJSON json.RawMessage) (Account, error) { - be := api.am.Backends(keystore.KeyStoreType) - - if len(be) == 0 { - return Account{}, errors.New("password based accounts not supported") - } - res, err := api.UI.ApproveImport(&ImportRequest{Meta: MetadataFromContext(ctx)}) - - if err != nil { - return Account{}, err - } - if !res.Approved { - return Account{}, ErrRequestDenied - } - acc, err := be[0].(*keystore.KeyStore).Import(keyJSON, res.OldPassword, res.NewPassword) - if err != nil { - api.UI.ShowError(err.Error()) - return Account{}, err - } - return Account{Typ: "Account", URL: acc.URL, Address: acc.Address}, nil -} - // Returns the external api version. This method does not require user acceptance. Available methods are // available via enumeration anyway, and this info does not contain user-specific data func (api *SignerAPI) Version(ctx context.Context) (string, error) { diff --git a/signer/core/api_test.go b/signer/core/api_test.go index 0e0c545178..bf4e91eb95 100644 --- a/signer/core/api_test.go +++ b/signer/core/api_test.go @@ -28,6 +28,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -47,6 +48,8 @@ func (ui *HeadlessUI) OnInputRequired(info UserInputRequest) (UserInputResponse, func (ui *HeadlessUI) OnSignerStartup(info StartupInfo) { } +func (ui *HeadlessUI) RegisterUIServer(api *UIServerAPI) { +} func (ui *HeadlessUI) OnApprovedTx(tx ethapi.SignTransactionResult) { fmt.Printf("OnApproved()\n") @@ -91,7 +94,7 @@ func (ui *HeadlessUI) ApproveListing(request *ListRequest) (ListResponse, error) case "A": return ListResponse{request.Accounts}, nil case "1": - l := make([]Account, 1) + l := make([]accounts.Account, 1) l[0] = request.Accounts[1] return ListResponse{l}, nil default: @@ -138,13 +141,8 @@ func setup(t *testing.T) (*SignerAPI, chan string) { } var ( ui = &HeadlessUI{controller} - api = NewSignerAPI( - 1, - tmpDirName(t), - true, - ui, - db, - true, true) + am = StartClefAccountManager(tmpDirName(t), true, true) + api = NewSignerAPI(am, 1337, true, ui, db, true) ) return api, controller } @@ -169,22 +167,22 @@ func failCreateAccountWithPassword(control chan string, api *SignerAPI, password control <- "Y" control <- password - acc, err := api.New(context.Background()) + addr, err := api.New(context.Background()) if err == nil { t.Fatal("Should have returned an error") } - if acc.Address != (common.Address{}) { + if addr != (common.Address{}) { t.Fatal("Empty address should be returned") } } func failCreateAccount(control chan string, api *SignerAPI, t *testing.T) { control <- "N" - acc, err := api.New(context.Background()) + addr, err := api.New(context.Background()) if err != ErrRequestDenied { t.Fatal(err) } - if acc.Address != (common.Address{}) { + if addr != (common.Address{}) { t.Fatal("Empty address should be returned") } } diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index d64ba1ef95..9593ad7a53 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -18,9 +18,7 @@ package core import ( "context" - "encoding/json" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -40,7 +38,7 @@ func (l *AuditLogger) List(ctx context.Context) ([]common.Address, error) { return res, e } -func (l *AuditLogger) New(ctx context.Context) (accounts.Account, error) { +func (l *AuditLogger) New(ctx context.Context) (common.Address, error) { return l.api.New(ctx) } @@ -86,15 +84,6 @@ func (l *AuditLogger) EcRecover(ctx context.Context, data hexutil.Bytes, sig hex return b, e } -func (l *AuditLogger) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) { - l.log.Info("Export", "type", "request", "metadata", MetadataFromContext(ctx).String(), - "addr", addr.Hex()) - j, e := l.api.Export(ctx, addr) - // In this case, we don't actually log the json-response, which may be extra sensitive - l.log.Info("Export", "type", "response", "json response size", len(j), "error", e) - return j, e -} - func (l *AuditLogger) Version(ctx context.Context) (string, error) { l.log.Info("Version", "type", "request", "metadata", MetadataFromContext(ctx).String()) data, err := l.api.Version(ctx) diff --git a/signer/core/cliui.go b/signer/core/cliui.go index 3ffc6b1916..a6c0bdb162 100644 --- a/signer/core/cliui.go +++ b/signer/core/cliui.go @@ -39,6 +39,10 @@ func NewCommandlineUI() *CommandlineUI { return &CommandlineUI{in: bufio.NewReader(os.Stdin)} } +func (ui *CommandlineUI) RegisterUIServer(api *UIServerAPI) { + // noop +} + // readString reads a single line from stdin, trimming if from spaces, enforcing // non-emptyness. func (ui *CommandlineUI) readString() string { @@ -223,7 +227,6 @@ func (ui *CommandlineUI) ApproveListing(request *ListRequest) (ListResponse, err for _, account := range request.Accounts { fmt.Printf(" [x] %v\n", account.Address.Hex()) fmt.Printf(" URL: %v\n", account.URL) - fmt.Printf(" Type: %v\n", account.Typ) } fmt.Printf("-------------------------------------------\n") showMetadata(request.Meta) diff --git a/signer/core/stdioui.go b/signer/core/stdioui.go index e169a8bc08..cb25bc2d00 100644 --- a/signer/core/stdioui.go +++ b/signer/core/stdioui.go @@ -32,12 +32,16 @@ type StdIOUI struct { } func NewStdIOUI() *StdIOUI { - log.Info("NewStdIOUI") client, err := rpc.DialContext(context.Background(), "stdio://") if err != nil { log.Crit("Could not create stdio client", "err", err) } - return &StdIOUI{client: *client} + ui := &StdIOUI{client: *client} + return ui +} + +func (ui *StdIOUI) RegisterUIServer(api *UIServerAPI) { + ui.client.RegisterName("clef", api) } // dispatch sends a request over the stdio diff --git a/signer/core/types.go b/signer/core/types.go index 8acfa7a6af..c1d3b9bf81 100644 --- a/signer/core/types.go +++ b/signer/core/types.go @@ -22,36 +22,11 @@ import ( "math/big" "strings" - "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" ) -type Accounts []Account - -func (as Accounts) String() string { - var output []string - for _, a := range as { - output = append(output, a.String()) - } - return strings.Join(output, "\n") -} - -type Account struct { - Typ string `json:"type"` - URL accounts.URL `json:"url"` - Address common.Address `json:"address"` -} - -func (a Account) String() string { - s, err := json.Marshal(a) - if err == nil { - return string(s) - } - return err.Error() -} - type ValidationInfo struct { Typ string `json:"type"` Message string `json:"message"` diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go new file mode 100644 index 0000000000..6dc68313b5 --- /dev/null +++ b/signer/core/uiapi.go @@ -0,0 +1,201 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . +// + +package core + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "math/big" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/crypto" +) + +// SignerUIAPI implements methods Clef provides for a UI to query, in the bidirectional communication +// channel. +// This API is considered secure, since a request can only +// ever arrive from the UI -- and the UI is capable of approving any action, thus we can consider these +// requests pre-approved. +// NB: It's very important that these methods are not ever exposed on the external service +// registry. +type UIServerAPI struct { + extApi *SignerAPI + am *accounts.Manager +} + +// NewUIServerAPI creates a new UIServerAPI +func NewUIServerAPI(extapi *SignerAPI) *UIServerAPI { + return &UIServerAPI{extapi, extapi.am} +} + +// List available accounts. As opposed to the external API definition, this method delivers +// the full Account object and not only Address. +// Example call +// {"jsonrpc":"2.0","method":"clef_listAccounts","params":[], "id":4} +func (s *UIServerAPI) ListAccounts(ctx context.Context) ([]accounts.Account, error) { + var accs []accounts.Account + for _, wallet := range s.am.Wallets() { + accs = append(accs, wallet.Accounts()...) + } + return accs, nil +} + +// rawWallet is a JSON representation of an accounts.Wallet interface, with its +// data contents extracted into plain fields. +type rawWallet struct { + URL string `json:"url"` + Status string `json:"status"` + Failure string `json:"failure,omitempty"` + Accounts []accounts.Account `json:"accounts,omitempty"` +} + +// ListWallets will return a list of wallets that clef manages +// Example call +// {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5} +func (s *UIServerAPI) ListWallets() []rawWallet { + wallets := make([]rawWallet, 0) // return [] instead of nil if empty + for _, wallet := range s.am.Wallets() { + status, failure := wallet.Status() + + raw := rawWallet{ + URL: wallet.URL().String(), + Status: status, + Accounts: wallet.Accounts(), + } + if failure != nil { + raw.Failure = failure.Error() + } + wallets = append(wallets, raw) + } + return wallets +} + +// DeriveAccount requests a HD wallet to derive a new account, optionally pinning +// it for later reuse. +// Example call +// {"jsonrpc":"2.0","method":"clef_deriveAccount","params":["ledger://","m/44'/60'/0'", false], "id":6} +func (s *UIServerAPI) DeriveAccount(url string, path string, pin *bool) (accounts.Account, error) { + wallet, err := s.am.Wallet(url) + if err != nil { + return accounts.Account{}, err + } + derivPath, err := accounts.ParseDerivationPath(path) + if err != nil { + return accounts.Account{}, err + } + if pin == nil { + pin = new(bool) + } + return wallet.Derive(derivPath, *pin) +} + +// fetchKeystore retrives the encrypted keystore from the account manager. +func fetchKeystore(am *accounts.Manager) *keystore.KeyStore { + return am.Backends(keystore.KeyStoreType)[0].(*keystore.KeyStore) +} + +// ImportRawKey stores the given hex encoded ECDSA key into the key directory, +// encrypting it with the passphrase. +// Example call (should fail on password too short) +// {"jsonrpc":"2.0","method":"clef_importRawKey","params":["1111111111111111111111111111111111111111111111111111111111111111","test"], "id":6} +func (s *UIServerAPI) ImportRawKey(privkey string, password string) (accounts.Account, error) { + key, err := crypto.HexToECDSA(privkey) + if err != nil { + return accounts.Account{}, err + } + if err := ValidatePasswordFormat(password); err != nil { + return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err) + } + // No error + return fetchKeystore(s.am).ImportECDSA(key, password) +} + +// OpenWallet initiates a hardware wallet opening procedure, establishing a USB +// connection and attempting to authenticate via the provided passphrase. Note, +// the method may return an extra challenge requiring a second open (e.g. the +// Trezor PIN matrix challenge). +// Example +// {"jsonrpc":"2.0","method":"clef_openWallet","params":["ledger://",""], "id":6} +func (s *UIServerAPI) OpenWallet(url string, passphrase *string) error { + wallet, err := s.am.Wallet(url) + if err != nil { + return err + } + pass := "" + if passphrase != nil { + pass = *passphrase + } + return wallet.Open(pass) +} + +// ChainId returns the chainid in use for Eip-155 replay protection +// Example call +// {"jsonrpc":"2.0","method":"clef_chainId","params":[], "id":8} +func (s *UIServerAPI) ChainId() math.HexOrDecimal64 { + return (math.HexOrDecimal64)(s.extApi.chainID.Uint64()) +} + +// SetChainId sets the chain id to use when signing transactions. +// Example call to set Ropsten: +// {"jsonrpc":"2.0","method":"clef_setChainId","params":["3"], "id":8} +func (s *UIServerAPI) SetChainId(id math.HexOrDecimal64) math.HexOrDecimal64 { + s.extApi.chainID = new(big.Int).SetUint64(uint64(id)) + return s.ChainId() +} + +// Export returns encrypted private key associated with the given address in web3 keystore format. +// Example +// {"jsonrpc":"2.0","method":"clef_export","params":["0x19e7e376e7c213b7e7e7e46cc70a5dd086daff2a"], "id":4} +func (s *UIServerAPI) Export(ctx context.Context, addr common.Address) (json.RawMessage, error) { + // Look up the wallet containing the requested signer + wallet, err := s.am.Find(accounts.Account{Address: addr}) + if err != nil { + return nil, err + } + if wallet.URL().Scheme != keystore.KeyStoreScheme { + return nil, fmt.Errorf("Account is not a keystore-account") + } + return ioutil.ReadFile(wallet.URL().Path) +} + +// Import tries to import the given keyJSON in the local keystore. The keyJSON data is expected to be +// in web3 keystore format. It will decrypt the keyJSON with the given passphrase and on successful +// decryption it will encrypt the key with the given newPassphrase and store it in the keystore. +// Example (the address in question has privkey `11...11`): +// {"jsonrpc":"2.0","method":"clef_import","params":[{"address":"19e7e376e7c213b7e7e7e46cc70a5dd086daff2a","crypto":{"cipher":"aes-128-ctr","ciphertext":"33e4cd3756091d037862bb7295e9552424a391a6e003272180a455ca2a9fb332","cipherparams":{"iv":"b54b263e8f89c42bb219b6279fba5cce"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":262144,"p":1,"r":8,"salt":"e4ca94644fd30569c1b1afbbc851729953c92637b7fe4bb9840bbb31ffbc64a5"},"mac":"f4092a445c2b21c0ef34f17c9cd0d873702b2869ec5df4439a0c2505823217e7"},"id":"216c7eac-e8c1-49af-a215-fa0036f29141","version":3},"test","yaddayadda"], "id":4} +func (api *UIServerAPI) Import(ctx context.Context, keyJSON json.RawMessage, oldPassphrase, newPassphrase string) (accounts.Account, error) { + be := api.am.Backends(keystore.KeyStoreType) + + if len(be) == 0 { + return accounts.Account{}, errors.New("password based accounts not supported") + } + if err := ValidatePasswordFormat(newPassphrase); err != nil { + return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err) + } + return be[0].(*keystore.KeyStore).Import(keyJSON, oldPassphrase, newPassphrase) +} + +// Other methods to be added, not yet implemented are: +// - Ruleset interaction: add rules, attest rulefiles +// - Store metadata about accounts, e.g. naming of accounts diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 07c34db220..1f81e21bd0 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -46,16 +46,16 @@ func consoleOutput(call otto.FunctionCall) otto.Value { return otto.Value{} } -// rulesetUI provides an implementation of SignerUI that evaluates a javascript +// rulesetUI provides an implementation of UIClientAPI that evaluates a javascript // file for each defined UI-method type rulesetUI struct { - next core.SignerUI // The next handler, for manual processing + next core.UIClientAPI // The next handler, for manual processing storage storage.Storage credentials storage.Storage jsRules string // The rules to use } -func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) { +func NewRuleEvaluator(next core.UIClientAPI, jsbackend, credentialsBackend storage.Storage) (*rulesetUI, error) { c := &rulesetUI{ next: next, storage: jsbackend, @@ -65,6 +65,9 @@ func NewRuleEvaluator(next core.SignerUI, jsbackend, credentialsBackend storage. return c, nil } +func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) { + // TODO, make it possible to query from js +} func (r *rulesetUI) Init(javascriptRules string) error { r.jsRules = javascriptRules diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index d3b2edd551..826d1ee334 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -77,6 +77,8 @@ type alwaysDenyUI struct{} func (alwaysDenyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { return core.UserInputResponse{}, nil } +func (alwaysDenyUI) RegisterUIServer(api *core.UIServerAPI) { +} func (alwaysDenyUI) OnSignerStartup(info core.StartupInfo) { } @@ -133,11 +135,11 @@ func initRuleEngine(js string) (*rulesetUI, error) { } func TestListRequest(t *testing.T) { - accs := make([]core.Account, 5) + accs := make([]accounts.Account, 5) for i := range accs { addr := fmt.Sprintf("000000000000000000000000000000000000000%x", i) - acc := core.Account{ + acc := accounts.Account{ Address: common.BytesToAddress(common.Hex2Bytes(addr)), URL: accounts.URL{Scheme: "test", Path: fmt.Sprintf("acc-%d", i)}, } @@ -208,6 +210,10 @@ type dummyUI struct { calls []string } +func (d *dummyUI) RegisterUIServer(api *core.UIServerAPI) { + panic("implement me") +} + func (d *dummyUI) OnInputRequired(info core.UserInputRequest) (core.UserInputResponse, error) { d.calls = append(d.calls, "OnInputRequired") return core.UserInputResponse{}, nil @@ -531,6 +537,8 @@ func (d *dontCallMe) OnInputRequired(info core.UserInputRequest) (core.UserInput d.t.Fatalf("Did not expect next-handler to be called") return core.UserInputResponse{}, nil } +func (d *dontCallMe) RegisterUIServer(api *core.UIServerAPI) { +} func (d *dontCallMe) OnSignerStartup(info core.StartupInfo) { } From 8771fbf3c8166ccd4ad98d3719604f9a2e8d6492 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 12 Feb 2019 18:05:28 +0100 Subject: [PATCH 17/22] rpc: make stdio usable over custom channels (#19046) --- rpc/stdio.go | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/rpc/stdio.go b/rpc/stdio.go index 8f6b7bd4bf..d5dc066c99 100644 --- a/rpc/stdio.go +++ b/rpc/stdio.go @@ -19,6 +19,7 @@ package rpc import ( "context" "errors" + "io" "net" "os" "time" @@ -26,19 +27,30 @@ import ( // DialStdIO creates a client on stdin/stdout. func DialStdIO(ctx context.Context) (*Client, error) { + return DialIO(ctx, os.Stdin, os.Stdout) +} + +// DialIO creates a client which uses the given IO channels +func DialIO(ctx context.Context, in io.Reader, out io.Writer) (*Client, error) { return newClient(ctx, func(_ context.Context) (ServerCodec, error) { - return NewJSONCodec(stdioConn{}), nil + return NewJSONCodec(stdioConn{ + in: in, + out: out, + }), nil }) } -type stdioConn struct{} +type stdioConn struct { + in io.Reader + out io.Writer +} func (io stdioConn) Read(b []byte) (n int, err error) { - return os.Stdin.Read(b) + return io.in.Read(b) } func (io stdioConn) Write(b []byte) (n int, err error) { - return os.Stdout.Write(b) + return io.out.Write(b) } func (io stdioConn) Close() error { From b30109df3c7c56cb0d1752fc03f478474c3c190a Mon Sep 17 00:00:00 2001 From: gluk256 Date: Wed, 13 Feb 2019 03:12:41 +0400 Subject: [PATCH 18/22] swarm/pss: mutex lifecycle fixed (#19045) --- swarm/pss/protocol.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/swarm/pss/protocol.go b/swarm/pss/protocol.go index 5fcae090ef..5f47ee47d1 100644 --- a/swarm/pss/protocol.go +++ b/swarm/pss/protocol.go @@ -228,6 +228,7 @@ func ToP2pMsg(msg []byte) (p2p.Msg, error) { // to link the peer to. // The key must exist in the pss store prior to adding the peer. func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key string) (p2p.MsgReadWriter, error) { + var ok bool rw := &PssReadWriter{ Pss: p.Pss, rw: make(chan p2p.Msg), @@ -242,19 +243,21 @@ func (p *Protocol) AddPeer(peer *p2p.Peer, topic Topic, asymmetric bool, key str } if asymmetric { p.Pss.pubKeyPoolMu.Lock() - if _, ok := p.Pss.pubKeyPool[key]; !ok { + _, ok = p.Pss.pubKeyPool[key] + p.Pss.pubKeyPoolMu.Unlock() + if !ok { return nil, fmt.Errorf("asym key does not exist: %s", key) } - p.Pss.pubKeyPoolMu.Unlock() p.RWPoolMu.Lock() p.pubKeyRWPool[key] = rw p.RWPoolMu.Unlock() } else { p.Pss.symKeyPoolMu.Lock() - if _, ok := p.Pss.symKeyPool[key]; !ok { + _, ok = p.Pss.symKeyPool[key] + p.Pss.symKeyPoolMu.Unlock() + if !ok { return nil, fmt.Errorf("symkey does not exist: %s", key) } - p.Pss.symKeyPoolMu.Unlock() p.RWPoolMu.Lock() p.symKeyRWPool[key] = rw p.RWPoolMu.Unlock() From 3d22a46c94f1d842dbada665b36a453362adda74 Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 12 Feb 2019 18:17:44 -0500 Subject: [PATCH 19/22] swarm/storage: fix HashExplore concurrency bug ethersphere#1211 (#19028) * swarm/storage: fix HashExplore concurrency bug ethersphere#1211 * swarm/storage: lock as value not pointer * swarm/storage: wait for to complete * swarm/storage: fix linter problems * swarm/storage: append to nil slice --- swarm/storage/filestore.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/swarm/storage/filestore.go b/swarm/storage/filestore.go index 4240f5b1c5..0bad944ee9 100644 --- a/swarm/storage/filestore.go +++ b/swarm/storage/filestore.go @@ -20,6 +20,7 @@ import ( "context" "io" "sort" + "sync" ) /* @@ -101,38 +102,45 @@ func (f *FileStore) HashSize() int { // GetAllReferences is a public API. This endpoint returns all chunk hashes (only) for a given file func (f *FileStore) GetAllReferences(ctx context.Context, data io.Reader, toEncrypt bool) (addrs AddressCollection, err error) { // create a special kind of putter, which only will store the references - putter := &HashExplorer{ + putter := &hashExplorer{ hasherStore: NewHasherStore(f.ChunkStore, f.hashFunc, toEncrypt), - References: make([]Reference, 0), } // do the actual splitting anyway, no way around it - _, _, err = PyramidSplit(ctx, data, putter, putter) + _, wait, err := PyramidSplit(ctx, data, putter, putter) + if err != nil { + return nil, err + } + // wait for splitting to be complete and all chunks processed + err = wait(ctx) if err != nil { return nil, err } // collect all references addrs = NewAddressCollection(0) - for _, ref := range putter.References { + for _, ref := range putter.references { addrs = append(addrs, Address(ref)) } sort.Sort(addrs) return addrs, nil } -// HashExplorer is a special kind of putter which will only store chunk references -type HashExplorer struct { +// hashExplorer is a special kind of putter which will only store chunk references +type hashExplorer struct { *hasherStore - References []Reference + references []Reference + lock sync.Mutex } // HashExplorer's Put will add just the chunk hashes to its `References` -func (he *HashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) { +func (he *hashExplorer) Put(ctx context.Context, chunkData ChunkData) (Reference, error) { // Need to do the actual Put, which returns the references ref, err := he.hasherStore.Put(ctx, chunkData) if err != nil { return nil, err } // internally store the reference - he.References = append(he.References, ref) + he.lock.Lock() + he.references = append(he.references, ref) + he.lock.Unlock() return ref, nil } From 555b3652dcb41b67cbdee50713230f5be459e144 Mon Sep 17 00:00:00 2001 From: Dan Kinsley Date: Tue, 12 Feb 2019 16:23:10 -0700 Subject: [PATCH 20/22] accounts/abi/bind/backends: add TransactionByHash to SimulatedBackend (#19026) --- accounts/abi/bind/backends/simulated.go | 19 ++++++ accounts/abi/bind/backends/simulated_test.go | 66 ++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 accounts/abi/bind/backends/simulated_test.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 6f46fc1495..400227c3a8 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -164,6 +164,25 @@ func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common return receipt, nil } +// TransactionByHash checks the pool of pending transactions in addition to the +// blockchain. The isPending return value indicates whether the transaction has been +// mined yet. Note that the transaction may not be part of the canonical chain even if +// it's not pending. +func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) { + + tx = b.pendingBlock.Transaction(txHash) + if tx != nil { + return tx, true, nil + } + + tx, _, _, _ = rawdb.ReadTransaction(b.database, txHash) + if tx != nil { + return tx, false, nil + } + + return nil, false, ethereum.NotFound +} + // PendingCodeAt returns the code associated with an account in the pending state. func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { b.mu.Lock() diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go new file mode 100644 index 0000000000..ac6e9e228a --- /dev/null +++ b/accounts/abi/bind/backends/simulated_test.go @@ -0,0 +1,66 @@ +package backends_test + +import ( + "context" + "math/big" + "testing" + + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +func TestSimulatedBackend(t *testing.T) { + var gasLimit uint64 = 8000029 + key, _ := crypto.GenerateKey() // nolint: gosec + auth := bind.NewKeyedTransactor(key) + genAlloc := make(core.GenesisAlloc) + genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)} + + sim := backends.NewSimulatedBackend(genAlloc, gasLimit) + + // should return an error if the tx is not found + txHash := common.HexToHash("2") + _, isPending, err := sim.TransactionByHash(context.Background(), txHash) + + if isPending { + t.Fatal("transaction should not be pending") + } + if err != ethereum.NotFound { + t.Fatalf("err should be `ethereum.NotFound` but received %v", err) + } + + // generate a transaction and confirm you can retrieve it + code := `6060604052600a8060106000396000f360606040526008565b00` + var gas uint64 = 3000000 + tx := types.NewContractCreation(0, big.NewInt(0), gas, big.NewInt(1), common.FromHex(code)) + tx, _ = types.SignTx(tx, types.HomesteadSigner{}, key) + + err = sim.SendTransaction(context.Background(), tx) + if err != nil { + t.Fatal("error sending transaction") + } + + txHash = tx.Hash() + _, isPending, err = sim.TransactionByHash(context.Background(), txHash) + if err != nil { + t.Fatalf("error getting transaction with hash: %v", txHash.String()) + } + if !isPending { + t.Fatal("transaction should have pending status") + } + + sim.Commit() + tx, isPending, err = sim.TransactionByHash(context.Background(), txHash) + if err != nil { + t.Fatalf("error getting transaction with hash: %v", txHash.String()) + } + if isPending { + t.Fatal("transaction should not have pending status") + } + +} From d596bea2d501d20b92e0fd4baa8bba682157dfa7 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 13 Feb 2019 14:15:03 +0700 Subject: [PATCH 21/22] swarm: fix uptime gauge update goroutine leak by introducing cleanup functions (#19040) --- swarm/swarm.go | 53 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/swarm/swarm.go b/swarm/swarm.go index 705fc43975..5b0e5f177c 100644 --- a/swarm/swarm.go +++ b/swarm/swarm.go @@ -79,7 +79,7 @@ type Swarm struct { swap *swap.Swap stateStore *state.DBStore accountingMetrics *protocols.AccountingMetrics - startTime time.Time + cleanupFuncs []func() error tracerClose io.Closer } @@ -106,9 +106,10 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e } self = &Swarm{ - config: config, - backend: backend, - privateKey: config.ShiftPrivateKey(), + config: config, + backend: backend, + privateKey: config.ShiftPrivateKey(), + cleanupFuncs: []func() error{}, } log.Debug("Setting up Swarm service components") @@ -344,7 +345,7 @@ Start is called when the stack is started */ // implements the node.Service interface func (self *Swarm) Start(srv *p2p.Server) error { - self.startTime = time.Now() + startTime := time.Now() self.tracerClose = tracing.Closer @@ -396,28 +397,30 @@ func (self *Swarm) Start(srv *p2p.Server) error { }() } - self.periodicallyUpdateGauges() + doneC := make(chan struct{}) + + self.cleanupFuncs = append(self.cleanupFuncs, func() error { + close(doneC) + return nil + }) + + go func(time.Time) { + for { + select { + case <-time.After(updateGaugesPeriod): + uptimeGauge.Update(time.Since(startTime).Nanoseconds()) + requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen())) + case <-doneC: + return + } + } + }(startTime) startCounter.Inc(1) self.streamer.Start(srv) return nil } -func (self *Swarm) periodicallyUpdateGauges() { - ticker := time.NewTicker(updateGaugesPeriod) - - go func() { - for range ticker.C { - self.updateGauges() - } - }() -} - -func (self *Swarm) updateGauges() { - uptimeGauge.Update(time.Since(self.startTime).Nanoseconds()) - requestsCacheGauge.Update(int64(self.netStore.RequestsCacheLen())) -} - // implements the node.Service interface // stops all component services. func (self *Swarm) Stop() error { @@ -452,6 +455,14 @@ func (self *Swarm) Stop() error { if self.stateStore != nil { self.stateStore.Close() } + + for _, cleanF := range self.cleanupFuncs { + err = cleanF() + if err != nil { + log.Error("encountered an error while running cleanup function", "err", err) + break + } + } return err } From 3fd6db2bf63ce90232de445c7f33943406a5e634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Wed, 13 Feb 2019 13:03:23 +0100 Subject: [PATCH 22/22] swarm: fix network/stream data races (#19051) * swarm/network/stream: newStreamerTester cleanup only if err is nil * swarm/network/stream: raise newStreamerTester waitForPeers timeout * swarm/network/stream: fix data races in GetPeerSubscriptions * swarm/storage: prevent data race on LDBStore.batchesC https://github.com/ethersphere/go-ethereum/issues/1198#issuecomment-461775049 * swarm/network/stream: fix TestGetSubscriptionsRPC data race https://github.com/ethersphere/go-ethereum/issues/1198#issuecomment-461768477 * swarm/network/stream: correctly use Simulation.Run callback https://github.com/ethersphere/go-ethereum/issues/1198#issuecomment-461783804 * swarm/network: protect addrCountC in Kademlia.AddrCountC function https://github.com/ethersphere/go-ethereum/issues/1198#issuecomment-462273444 * p2p/simulations: fix a deadlock calling getRandomNode with lock https://github.com/ethersphere/go-ethereum/issues/1198#issuecomment-462317407 * swarm/network/stream: terminate disconnect goruotines in tests * swarm/network/stream: reduce memory consumption when testing data races * swarm/network/stream: add watchDisconnections helper function * swarm/network/stream: add concurrent counter for tests * swarm/network/stream: rename race/norace test files and use const * swarm/network/stream: remove watchSim and its panic * swarm/network/stream: pass context in watchDisconnections * swarm/network/stream: add concurrent safe bool for watchDisconnections * swarm/storage: fix LDBStore.batchesC data race by not closing it --- p2p/simulations/network.go | 2 +- swarm/network/kademlia.go | 3 + swarm/network/stream/common_test.go | 62 ++++++++++++++- swarm/network/stream/delivery_test.go | 75 +++++-------------- swarm/network/stream/intervals_test.go | 30 ++------ swarm/network/stream/lightnode_test.go | 8 +- swarm/network/stream/norace_test.go | 24 ++++++ swarm/network/stream/race_test.go | 23 ++++++ swarm/network/stream/snapshot_sync_test.go | 58 ++++++++------ swarm/network/stream/stream.go | 6 ++ swarm/network/stream/streamer_test.go | 70 +++++++++++------ swarm/network/stream/syncer_test.go | 65 +++++++++------- .../visualized_snapshot_sync_sim_test.go | 44 +++-------- swarm/storage/ldbstore.go | 1 - 14 files changed, 274 insertions(+), 197 deletions(-) create mode 100644 swarm/network/stream/norace_test.go create mode 100644 swarm/network/stream/race_test.go diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index c2a3b9647f..d99633c9d7 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -461,7 +461,7 @@ func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node { if l == 0 { return nil } - return net.GetNode(filtered[rand.Intn(l)]) + return net.getNode(filtered[rand.Intn(l)]) } func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID { diff --git a/swarm/network/kademlia.go b/swarm/network/kademlia.go index 9f4245603f..1193e3b652 100644 --- a/swarm/network/kademlia.go +++ b/swarm/network/kademlia.go @@ -353,6 +353,9 @@ func (k *Kademlia) sendNeighbourhoodDepthChange() { // Not receiving from the returned channel will block Register function // when address count value changes. func (k *Kademlia) AddrCountC() <-chan int { + k.lock.Lock() + defer k.lock.Unlock() + if k.addrCountC == nil { k.addrCountC = make(chan int) } diff --git a/swarm/network/stream/common_test.go b/swarm/network/stream/common_test.go index 8a7d851fb7..afd08d2754 100644 --- a/swarm/network/stream/common_test.go +++ b/swarm/network/stream/common_test.go @@ -151,7 +151,7 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste // temp datadir datadir, err := ioutil.TempDir("", "streamer") if err != nil { - return nil, nil, nil, func() {}, err + return nil, nil, nil, nil, err } removeDataDir := func() { os.RemoveAll(datadir) @@ -163,12 +163,14 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste localStore, err := storage.NewTestLocalStoreForAddr(params) if err != nil { - return nil, nil, nil, removeDataDir, err + removeDataDir() + return nil, nil, nil, nil, err } netStore, err := storage.NewNetStore(localStore, nil) if err != nil { - return nil, nil, nil, removeDataDir, err + removeDataDir() + return nil, nil, nil, nil, err } delivery := NewDelivery(to, netStore) @@ -180,8 +182,9 @@ func newStreamerTester(registryOptions *RegistryOptions) (*p2ptest.ProtocolTeste } protocolTester := p2ptest.NewProtocolTester(addr.ID(), 1, streamer.runProtocol) - err = waitForPeers(streamer, 1*time.Second, 1) + err = waitForPeers(streamer, 10*time.Second, 1) if err != nil { + teardown() return nil, nil, nil, nil, errors.New("timeout: peer is not created") } @@ -317,3 +320,54 @@ func createTestLocalStorageForID(id enode.ID, addr *network.BzzAddr) (storage.Ch } return store, datadir, nil } + +// watchDisconnections receives simulation peer events in a new goroutine and sets atomic value +// disconnected to true in case of a disconnect event. +func watchDisconnections(ctx context.Context, sim *simulation.Simulation) (disconnected *boolean) { + log.Debug("Watching for disconnections") + disconnections := sim.PeerEvents( + ctx, + sim.NodeIDs(), + simulation.NewPeerEventsFilter().Drop(), + ) + disconnected = new(boolean) + go func() { + for { + select { + case <-ctx.Done(): + return + case d := <-disconnections: + if d.Error != nil { + log.Error("peer drop event error", "node", d.NodeID, "peer", d.PeerID, "err", d.Error) + } else { + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) + } + disconnected.set(true) + } + } + }() + return disconnected +} + +// boolean is used to concurrently set +// and read a boolean value. +type boolean struct { + v bool + mu sync.RWMutex +} + +// set sets the value. +func (b *boolean) set(v bool) { + b.mu.Lock() + defer b.mu.Unlock() + + b.v = v +} + +// bool reads the value. +func (b *boolean) bool() bool { + b.mu.RLock() + defer b.mu.RUnlock() + + return b.v +} diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index cb7690f3ef..e5821df4f9 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "testing" "time" @@ -48,10 +47,10 @@ func TestStreamerRetrieveRequest(t *testing.T) { Syncing: SyncingDisabled, } tester, streamer, _, teardown, err := newStreamerTester(regOpts) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -100,10 +99,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchangeWithoutStore(t *testing.T) { Retrieval: RetrievalEnabled, Syncing: SyncingDisabled, //do no syncing }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -172,10 +171,10 @@ func TestStreamerUpstreamRetrieveRequestMsgExchange(t *testing.T) { Retrieval: RetrievalEnabled, Syncing: SyncingDisabled, }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -362,10 +361,10 @@ func TestStreamerDownstreamChunkDeliveryMsgExchange(t *testing.T) { Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) { return &testClient{ @@ -485,7 +484,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) } log.Info("Starting simulation") - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() //determine the pivot node to be the first node of the simulation @@ -548,27 +548,10 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) retErrC <- err }() - log.Debug("Watching for disconnections") - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - - var disconnected atomic.Value - go func() { - for d := range disconnections { - if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - disconnected.Store(true) - } - } - }() + disconnected := watchDisconnections(ctx, sim) defer func() { - if err != nil { - if yes, ok := disconnected.Load().(bool); ok && yes { - err = errors.New("disconnect events received") - } + if err != nil && disconnected.bool() { + err = errors.New("disconnect events received") } }() @@ -589,7 +572,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, chunkCount int, skipCheck bool) return fmt.Errorf("Test failed, chunks not available on all nodes") } if err := <-retErrC; err != nil { - t.Fatalf("requesting chunks: %v", err) + return fmt.Errorf("requesting chunks: %v", err) } log.Debug("Test terminated successfully") return nil @@ -657,21 +640,22 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b b.Fatal(err) } - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { nodeIDs := sim.UpNodeIDs() node := nodeIDs[len(nodeIDs)-1] item, ok := sim.NodeItem(node, bucketKeyFileStore) if !ok { - b.Fatal("No filestore") + return errors.New("No filestore") } remoteFileStore := item.(*storage.FileStore) pivotNode := nodeIDs[0] item, ok = sim.NodeItem(pivotNode, bucketKeyNetStore) if !ok { - b.Fatal("No filestore") + return errors.New("No filestore") } netStore := item.(*storage.NetStore) @@ -679,26 +663,10 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b return err } - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - - var disconnected atomic.Value - go func() { - for d := range disconnections { - if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - disconnected.Store(true) - } - } - }() + disconnected := watchDisconnections(ctx, sim) defer func() { - if err != nil { - if yes, ok := disconnected.Load().(bool); ok && yes { - err = errors.New("disconnect events received") - } + if err != nil && disconnected.bool() { + err = errors.New("disconnect events received") } }() // benchmark loop @@ -713,12 +681,12 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b ctx := context.TODO() hash, wait, err := remoteFileStore.Store(ctx, testutil.RandomReader(i, chunkSize), int64(chunkSize), false) if err != nil { - b.Fatalf("expected no error. got %v", err) + return fmt.Errorf("store: %v", err) } // wait until all chunks stored err = wait(ctx) if err != nil { - b.Fatalf("expected no error. got %v", err) + return fmt.Errorf("wait store: %v", err) } // collect the hashes hashes[i] = hash @@ -754,10 +722,7 @@ func benchmarkDeliveryFromNodes(b *testing.B, nodes, chunkCount int, skipCheck b break Loop } } - if err != nil { - b.Fatal(err) - } - return nil + return err }) if result.Error != nil { b.Fatal(result.Error) diff --git a/swarm/network/stream/intervals_test.go b/swarm/network/stream/intervals_test.go index 248ba0c846..009a941ef4 100644 --- a/swarm/network/stream/intervals_test.go +++ b/swarm/network/stream/intervals_test.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "sync" - "sync/atomic" "testing" "time" @@ -118,13 +117,11 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { _, wait, err := fileStore.Store(ctx, testutil.RandomReader(1, size), int64(size), false) if err != nil { - log.Error("Store error: %v", "err", err) - t.Fatal(err) + return fmt.Errorf("store: %v", err) } err = wait(ctx) if err != nil { - log.Error("Wait error: %v", "err", err) - t.Fatal(err) + return fmt.Errorf("wait store: %v", err) } item, ok = sim.NodeItem(checker, bucketKeyRegistry) @@ -136,32 +133,15 @@ func testIntervals(t *testing.T, live bool, history *Range, skipCheck bool) { liveErrC := make(chan error) historyErrC := make(chan error) - log.Debug("Watching for disconnections") - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - err = registry.Subscribe(storer, NewStream(externalStreamName, "", live), history, Top) if err != nil { return err } - var disconnected atomic.Value - go func() { - for d := range disconnections { - if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - disconnected.Store(true) - } - } - }() + disconnected := watchDisconnections(ctx, sim) defer func() { - if err != nil { - if yes, ok := disconnected.Load().(bool); ok && yes { - err = errors.New("disconnect events received") - } + if err != nil && disconnected.bool() { + err = errors.New("disconnect events received") } }() diff --git a/swarm/network/stream/lightnode_test.go b/swarm/network/stream/lightnode_test.go index 043f2ea779..501660fab4 100644 --- a/swarm/network/stream/lightnode_test.go +++ b/swarm/network/stream/lightnode_test.go @@ -29,10 +29,10 @@ func TestLigthnodeRetrieveRequestWithRetrieve(t *testing.T) { Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(registryOptions) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -68,10 +68,10 @@ func TestLigthnodeRetrieveRequestWithoutRetrieve(t *testing.T) { Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(registryOptions) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -112,10 +112,10 @@ func TestLigthnodeRequestSubscriptionWithSync(t *testing.T) { Syncing: SyncingRegisterOnly, } tester, _, _, teardown, err := newStreamerTester(registryOptions) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] @@ -157,10 +157,10 @@ func TestLigthnodeRequestSubscriptionWithoutSync(t *testing.T) { Syncing: SyncingDisabled, } tester, _, _, teardown, err := newStreamerTester(registryOptions) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() node := tester.Nodes[0] diff --git a/swarm/network/stream/norace_test.go b/swarm/network/stream/norace_test.go new file mode 100644 index 0000000000..b324f6939c --- /dev/null +++ b/swarm/network/stream/norace_test.go @@ -0,0 +1,24 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !race + +package stream + +// Provide a flag to reduce the scope of tests when running them +// with race detector. Some of the tests are doing a lot of allocations +// on the heap, and race detector uses much more memory to track them. +const raceTest = false diff --git a/swarm/network/stream/race_test.go b/swarm/network/stream/race_test.go new file mode 100644 index 0000000000..8aed3542bf --- /dev/null +++ b/swarm/network/stream/race_test.go @@ -0,0 +1,23 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build race + +package stream + +// Reduce the scope of some tests when running with race detector, +// as it raises the memory consumption significantly. +const raceTest = true diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index c32ed7d077..b45d0aed50 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -17,11 +17,12 @@ package stream import ( "context" + "errors" "fmt" + "io/ioutil" "os" "runtime" "sync" - "sync/atomic" "testing" "time" @@ -92,6 +93,15 @@ func TestSyncingViaGlobalSync(t *testing.T) { if *longrunning { chnkCnt = []int{1, 8, 32, 256, 1024} nodeCnt = []int{16, 32, 64, 128, 256} + } else if raceTest { + // TestSyncingViaGlobalSync allocates a lot of memory + // with race detector. By reducing the number of chunks + // and nodes, memory consumption is lower and data races + // are still checked, while correctness of syncing is + // tested with more chunks and nodes in regular (!race) + // tests. + chnkCnt = []int{4} + nodeCnt = []int{16} } else { //default test chnkCnt = []int{4, 32} @@ -113,7 +123,23 @@ var simServiceMap = map[string]simulation.ServiceFunc{ return nil, nil, err } - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + var dir string + var store *state.DBStore + if raceTest { + // Use on-disk DBStore to reduce memory consumption in race tests. + dir, err = ioutil.TempDir("", "swarm-stream-") + if err != nil { + return nil, nil, err + } + store, err = state.NewDBStore(dir) + if err != nil { + return nil, nil, err + } + } else { + store = state.NewInmemoryStore() + } + + r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SyncUpdateDelay: 3 * time.Second, @@ -156,36 +182,24 @@ func testSyncingViaGlobalSync(t *testing.T, chunkCount int, nodeCount int) { t.Fatal(err) } - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - - var disconnected atomic.Value - go func() { - for d := range disconnections { - if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - disconnected.Store(true) - } - } - }() - result := runSim(conf, ctx, sim, chunkCount) if result.Error != nil { t.Fatal(result.Error) } - if yes, ok := disconnected.Load().(bool); ok && yes { - t.Fatal("disconnect events received") - } log.Info("Simulation ended") } func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulation, chunkCount int) simulation.Result { - return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + return sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { + disconnected := watchDisconnections(ctx, sim) + defer func() { + if err != nil && disconnected.bool() { + err = errors.New("disconnect events received") + } + }() + nodeIDs := sim.UpNodeIDs() for _, n := range nodeIDs { //get the kademlia overlay address from this ID diff --git a/swarm/network/stream/stream.go b/swarm/network/stream/stream.go index 5d3e23eb18..65bcce8b94 100644 --- a/swarm/network/stream/stream.go +++ b/swarm/network/stream/stream.go @@ -939,16 +939,22 @@ It returns a map of node IDs with an array of string representations of Stream o func (api *API) GetPeerSubscriptions() map[string][]string { //create the empty map pstreams := make(map[string][]string) + //iterate all streamer peers + api.streamer.peersMu.RLock() + defer api.streamer.peersMu.RUnlock() + for id, p := range api.streamer.peers { var streams []string //every peer has a map of stream servers //every stream server represents a subscription + p.serverMu.RLock() for s := range p.servers { //append the string representation of the stream //to the list for this peer streams = append(streams, s.String()) } + p.serverMu.RUnlock() //set the array of stream servers to the map pstreams[id.String()] = streams } diff --git a/swarm/network/stream/streamer_test.go b/swarm/network/stream/streamer_test.go index c2aee61b73..e92ee37834 100644 --- a/swarm/network/stream/streamer_test.go +++ b/swarm/network/stream/streamer_test.go @@ -41,10 +41,10 @@ import ( func TestStreamerSubscribe(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", true) err = streamer.Subscribe(tester.Nodes[0].ID(), stream, NewRange(0, 0), Top) @@ -55,10 +55,10 @@ func TestStreamerSubscribe(t *testing.T) { func TestStreamerRequestSubscription(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", false) err = streamer.RequestSubscription(tester.Nodes[0].ID(), stream, &Range{}, Top) @@ -146,10 +146,10 @@ func (self *testServer) Close() { func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterClientFunc("foo", func(p *Peer, t string, live bool) (Client, error) { return newTestClient(t), nil @@ -239,10 +239,10 @@ func TestStreamerDownstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", false) @@ -306,10 +306,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchange(t *testing.T) { func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", true) @@ -372,10 +372,10 @@ func TestStreamerUpstreamSubscribeUnsubscribeMsgExchangeLive(t *testing.T) { func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { return newTestServer(t, 0), nil @@ -416,10 +416,10 @@ func TestStreamerUpstreamSubscribeErrorMsgExchange(t *testing.T) { func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", true) @@ -479,10 +479,10 @@ func TestStreamerUpstreamSubscribeLiveAndHistory(t *testing.T) { func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", true) @@ -544,10 +544,10 @@ func TestStreamerDownstreamCorruptHashesMsgExchange(t *testing.T) { func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() stream := NewStream("foo", "", true) @@ -643,10 +643,10 @@ func TestStreamerDownstreamOfferedHashesMsgExchange(t *testing.T) { func TestStreamerRequestSubscriptionQuitMsgExchange(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(nil) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { return newTestServer(t, 10), nil @@ -780,10 +780,10 @@ func TestMaxPeerServersWithUnsubscribe(t *testing.T) { Syncing: SyncingDisabled, MaxPeerServers: maxPeerServers, }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { return newTestServer(t, 0), nil @@ -854,10 +854,10 @@ func TestMaxPeerServersWithoutUnsubscribe(t *testing.T) { tester, streamer, _, teardown, err := newStreamerTester(&RegistryOptions{ MaxPeerServers: maxPeerServers, }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() streamer.RegisterServerFunc("foo", func(p *Peer, t string, live bool) (Server, error) { return newTestServer(t, 0), nil @@ -940,10 +940,10 @@ func TestHasPriceImplementation(t *testing.T) { Retrieval: RetrievalDisabled, Syncing: SyncingDisabled, }) - defer teardown() if err != nil { t.Fatal(err) } + defer teardown() if r.prices == nil { t.Fatal("No prices implementation available for the stream protocol") @@ -1177,6 +1177,7 @@ starts the simulation, waits for SyncUpdateDelay in order to kick off stream registration, then tests that there are subscriptions. */ func TestGetSubscriptionsRPC(t *testing.T) { + // arbitrarily set to 4 nodeCount := 4 // run with more nodes if `longrunning` flag is set @@ -1188,19 +1189,16 @@ func TestGetSubscriptionsRPC(t *testing.T) { // holds the msg code for SubscribeMsg var subscribeMsgCode uint64 var ok bool - var expectedMsgCount = 0 + var expectedMsgCount counter // this channel signalizes that the expected amount of subscriptiosn is done allSubscriptionsDone := make(chan struct{}) - lock := sync.RWMutex{} // after the test, we need to reset the subscriptionFunc to the default defer func() { subscriptionFunc = doRequestSubscription }() // we use this subscriptionFunc for this test: just increases count and calls the actual subscription subscriptionFunc = func(r *Registry, p *network.Peer, bin uint8, subs map[enode.ID]map[Stream]struct{}) bool { - lock.Lock() - expectedMsgCount++ - lock.Unlock() + expectedMsgCount.inc() doRequestSubscription(r, p, bin, subs) return true } @@ -1290,24 +1288,24 @@ func TestGetSubscriptionsRPC(t *testing.T) { select { case <-allSubscriptionsDone: case <-ctx.Done(): - t.Fatal("Context timed out") + return errors.New("Context timed out") } - log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount) + log.Debug("Expected message count: ", "expectedMsgCount", expectedMsgCount.count()) //now iterate again, this time we call each node via RPC to get its subscriptions realCount := 0 for _, node := range nodes { //create rpc client client, err := node.Client() if err != nil { - t.Fatalf("create node 1 rpc client fail: %v", err) + return fmt.Errorf("create node 1 rpc client fail: %v", err) } //ask it for subscriptions pstreams := make(map[string][]string) err = client.Call(&pstreams, "stream_getPeerSubscriptions") if err != nil { - t.Fatal(err) + return fmt.Errorf("client call stream_getPeerSubscriptions: %v", err) } //length of the subscriptions can not be smaller than number of peers log.Debug("node subscriptions", "node", node.String()) @@ -1324,8 +1322,9 @@ func TestGetSubscriptionsRPC(t *testing.T) { } } // every node is mutually subscribed to each other, so the actual count is half of it - if realCount/2 != expectedMsgCount { - return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, expectedMsgCount) + emc := expectedMsgCount.count() + if realCount/2 != emc { + return fmt.Errorf("Real subscriptions and expected amount don't match; real: %d, expected: %d", realCount/2, emc) } return nil }) @@ -1333,3 +1332,26 @@ func TestGetSubscriptionsRPC(t *testing.T) { t.Fatal(result.Error) } } + +// counter is used to concurrently increment +// and read an integer value. +type counter struct { + v int + mu sync.RWMutex +} + +// Increment the counter. +func (c *counter) inc() { + c.mu.Lock() + defer c.mu.Unlock() + + c.v++ +} + +// Read the counter value. +func (c *counter) count() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.v +} diff --git a/swarm/network/stream/syncer_test.go b/swarm/network/stream/syncer_test.go index 5656963d9c..be0752a9d0 100644 --- a/swarm/network/stream/syncer_test.go +++ b/swarm/network/stream/syncer_test.go @@ -22,8 +22,8 @@ import ( "fmt" "io/ioutil" "math" + "os" "sync" - "sync/atomic" "testing" "time" @@ -44,9 +44,15 @@ const dataChunkCount = 200 func TestSyncerSimulation(t *testing.T) { testSyncBetweenNodes(t, 2, dataChunkCount, true, 1) - testSyncBetweenNodes(t, 4, dataChunkCount, true, 1) - testSyncBetweenNodes(t, 8, dataChunkCount, true, 1) - testSyncBetweenNodes(t, 16, dataChunkCount, true, 1) + // This test uses much more memory when running with + // race detector. Allow it to finish successfully by + // reducing its scope, and still check for data races + // with the smallest number of nodes. + if !raceTest { + testSyncBetweenNodes(t, 4, dataChunkCount, true, 1) + testSyncBetweenNodes(t, 8, dataChunkCount, true, 1) + testSyncBetweenNodes(t, 16, dataChunkCount, true, 1) + } } func createMockStore(globalStore mock.GlobalStorer, id enode.ID, addr *network.BzzAddr) (lstore storage.ChunkStore, datadir string, err error) { @@ -80,7 +86,23 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p return nil, nil, err } - r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + var dir string + var store *state.DBStore + if raceTest { + // Use on-disk DBStore to reduce memory consumption in race tests. + dir, err = ioutil.TempDir("", "swarm-stream-") + if err != nil { + return nil, nil, err + } + store, err = state.NewDBStore(dir) + if err != nil { + return nil, nil, err + } + } else { + store = state.NewInmemoryStore() + } + + r := NewRegistry(addr.ID(), delivery, netStore, store, &RegistryOptions{ Retrieval: RetrievalDisabled, Syncing: SyncingAutoSubscribe, SkipCheck: skipCheck, @@ -89,6 +111,9 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p cleanup = func() { r.Close() clean() + if dir != "" { + os.RemoveAll(dir) + } } return r, cleanup, nil @@ -114,26 +139,10 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p nodeIndex[id] = i } - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - - var disconnected atomic.Value - go func() { - for d := range disconnections { - if d.Error != nil { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - disconnected.Store(true) - } - } - }() + disconnected := watchDisconnections(ctx, sim) defer func() { - if err != nil { - if yes, ok := disconnected.Load().(bool); ok && yes { - err = errors.New("disconnect events received") - } + if err != nil && disconnected.bool() { + err = errors.New("disconnect events received") } }() @@ -142,7 +151,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p id := nodeIDs[j] client, err := sim.Net.GetNode(id).Client() if err != nil { - t.Fatal(err) + return fmt.Errorf("node %s client: %v", id, err) } sid := nodeIDs[j+1] client.CallContext(ctx, nil, "stream_subscribeStream", sid, NewStream("SYNC", FormatSyncBinKey(1), false), NewRange(0, 0), Top) @@ -158,7 +167,7 @@ func testSyncBetweenNodes(t *testing.T, nodes, chunkCount int, skipCheck bool, p size := chunkCount * chunkSize _, wait, err := fileStore.Store(ctx, testutil.RandomReader(j, size), int64(size), false) if err != nil { - t.Fatal(err.Error()) + return fmt.Errorf("fileStore.Store: %v", err) } wait(ctx) } @@ -273,7 +282,7 @@ func TestSameVersionID(t *testing.T) { //the peers should connect, thus getting the peer should not return nil if registry.getPeer(nodes[1]) == nil { - t.Fatal("Expected the peer to not be nil, but it is") + return errors.New("Expected the peer to not be nil, but it is") } return nil }) @@ -338,7 +347,7 @@ func TestDifferentVersionID(t *testing.T) { //getting the other peer should fail due to the different version numbers if registry.getPeer(nodes[1]) != nil { - t.Fatal("Expected the peer to be nil, but it is not") + return errors.New("Expected the peer to be nil, but it is not") } return nil }) diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 3694dd3117..2e091f991b 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -66,31 +66,6 @@ func setupSim(serviceMap map[string]simulation.ServiceFunc) (int, int, *simulati return nodeCount, chunkCount, sim } -//watch for disconnections and wait for healthy -func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) { - ctx, cancelSimRun := context.WithTimeout(context.Background(), 1*time.Minute) - - if _, err := sim.WaitTillHealthy(ctx); err != nil { - panic(err) - } - - disconnections := sim.PeerEvents( - context.Background(), - sim.NodeIDs(), - simulation.NewPeerEventsFilter().Drop(), - ) - - go func() { - for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) - panic("unexpected disconnect") - cancelSimRun() - } - }() - - return ctx, cancelSimRun -} - //This test requests bogus hashes into the network func TestNonExistingHashesWithServer(t *testing.T) { @@ -102,19 +77,25 @@ func TestNonExistingHashesWithServer(t *testing.T) { panic(err) } - ctx, cancelSimRun := watchSim(sim) - defer cancelSimRun() - //in order to get some meaningful visualization, it is beneficial //to define a minimum duration of this test testDuration := 20 * time.Second - result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { + result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) (err error) { + disconnected := watchDisconnections(ctx, sim) + defer func() { + if err != nil { + if yes, ok := disconnected.Load().(bool); ok && yes { + err = errors.New("disconnect events received") + } + } + }() + //check on the node's FileStore (netstore) id := sim.Net.GetRandomUpNode().ID() item, ok := sim.NodeItem(id, bucketKeyFileStore) if !ok { - t.Fatalf("No filestore") + return errors.New("No filestore") } fileStore := item.(*storage.FileStore) //create a bogus hash @@ -213,9 +194,6 @@ func TestSnapshotSyncWithServer(t *testing.T) { panic(err) } - ctx, cancelSimRun := watchSim(sim) - defer cancelSimRun() - //run the sim result := runSim(conf, ctx, sim, chunkCount) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index a2f24eff06..f98809fc6c 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -1049,7 +1049,6 @@ func (s *LDBStore) Close() { s.lock.Unlock() // force writing out current batch s.writeCurrentBatch() - close(s.batchesC) s.db.Close() }