From 21dd59bd047d1f291c1a773b690825a3654ea969 Mon Sep 17 00:00:00 2001 From: Franklin Date: Tue, 20 Nov 2018 22:16:40 +0800 Subject: [PATCH 01/68] . --- core/blockchain.go | 6 +++++- node/node.go | 2 +- rpc/ipc.go | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d173b2de29..ee1c0702b5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -445,7 +445,11 @@ func (bc *BlockChain) repair(head **types.Block) error { return nil } // Otherwise rewind one block and recheck state availability there - (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) + if block == nil { + return fmt.Errorf("failed to repair block, can not get block at height %d", (*head).NumberU64()) + } + (*head) = block } } diff --git a/node/node.go b/node/node.go index 85299dba74..0a931a9dd5 100644 --- a/node/node.go +++ b/node/node.go @@ -322,7 +322,7 @@ func (n *Node) stopIPC() { n.ipcListener.Close() n.ipcListener = nil - n.log.Info("IPC endpoint closed", "endpoint", n.ipcEndpoint) + n.log.Info("IPC endpoint closed", "url", n.ipcEndpoint) } if n.ipcHandler != nil { n.ipcHandler.Stop() diff --git a/rpc/ipc.go b/rpc/ipc.go index b05e503d74..1467f7a0c9 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -24,17 +24,17 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" ) -// ServeListener accepts connections on l, serving JSON-RPC on them. +// ServeListener accepts connections on l, serving IPC-RPC on them. func (srv *Server) ServeListener(l net.Listener) error { for { conn, err := l.Accept() if netutil.IsTemporaryError(err) { - log.Warn("RPC accept error", "err", err) + log.Warn("IPC accept error", "err", err) continue } else if err != nil { return err } - log.Trace("Accepted connection", "addr", conn.RemoteAddr()) + log.Trace("IPC accepted connection") go srv.ServeCodec(NewJSONCodec(conn), OptionMethodInvocation|OptionSubscriptions) } } From 8b9f46941924a04ce1091532a74def03e70c7559 Mon Sep 17 00:00:00 2001 From: Liang Ma Date: Tue, 27 Nov 2018 10:52:30 +0000 Subject: [PATCH 02/68] p2p/protocols: fix minor comments typo (#18185) --- p2p/protocols/protocol.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/protocols/protocol.go b/p2p/protocols/protocol.go index 7dddd852fa..b16720dd39 100644 --- a/p2p/protocols/protocol.go +++ b/p2p/protocols/protocol.go @@ -381,7 +381,7 @@ func (p *Peer) handleIncoming(handle func(ctx context.Context, msg interface{}) // * arguments // * context // * the local handshake to be sent to the remote peer -// * funcion to be called on the remote handshake (can be nil) +// * function to be called on the remote handshake (can be nil) // * expects a remote handshake back of the same type // * the dialing peer needs to send the handshake first and then waits for remote // * the listening peer waits for the remote handshake and then sends it From 89fe24bbcc2395b9fbd449f5aeb9da8df073991d Mon Sep 17 00:00:00 2001 From: ANOTHEL Date: Tue, 27 Nov 2018 21:00:57 +0900 Subject: [PATCH 03/68] p2p/discv5: minor code simplification (#18188) * Update net.go more simple * Update net.go --- p2p/discv5/net.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index a6cabf0803..976eae7df3 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -567,12 +567,11 @@ loop: net.ticketStore.searchLookupDone(res.target, res.nodes, func(n *Node, topic Topic) []byte { if n.state != nil && n.state.canQuery { return net.conn.send(n, topicQueryPacket, topicQuery{Topic: topic}) // TODO: set expiration - } else { - if n.state == unknown { - net.ping(n, n.addr()) - } - return nil } + if n.state == unknown { + net.ping(n, n.addr()) + } + return nil }) case <-statsDump.C: From edc39aaedf526bfb7757a445c1f9dd42f45dc8d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 27 Nov 2018 14:50:47 +0200 Subject: [PATCH 04/68] p2p/discv5: gofmt --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index 976eae7df3..cdeb28dd5b 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -571,7 +571,7 @@ loop: if n.state == unknown { net.ping(n, n.addr()) } - return nil + return nil }) case <-statsDump.C: From bfed28a42187c7bf7b8f50401fe239278a4ed530 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Nov 2018 09:29:05 +0100 Subject: [PATCH 05/68] core: more detailed metrics for block processing (#18119) --- core/blockchain.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index bcd53d4987..a672c0ab64 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -47,7 +47,10 @@ import ( ) var ( - blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) + blockInsertTimer = metrics.NewRegisteredTimer("chain/inserts", nil) + blockValidationTimer = metrics.NewRegisteredTimer("chain/validation", nil) + blockExecutionTimer = metrics.NewRegisteredTimer("chain/execution", nil) + blockWriteTimer = metrics.NewRegisteredTimer("chain/write", nil) ErrNoGenesis = errors.New("Genesis not found in chain") ) @@ -1188,7 +1191,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] return it.index, events, coalescedLogs, err } // Process block using the parent state as reference point. + t0 := time.Now() receipts, logs, usedGas, err := bc.processor.Process(block, state, bc.vmConfig) + t1 := time.Now() if err != nil { bc.reportBlock(block, receipts, err) return it.index, events, coalescedLogs, err @@ -1198,13 +1203,19 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] bc.reportBlock(block, receipts, err) return it.index, events, coalescedLogs, err } + t2 := time.Now() proctime := time.Since(start) // Write the block to the chain and get the status. status, err := bc.WriteBlockWithState(block, receipts, state) + t3 := time.Now() if err != nil { return it.index, events, coalescedLogs, err } + blockInsertTimer.UpdateSince(start) + blockExecutionTimer.Update(t1.Sub(t0)) + blockValidationTimer.Update(t2.Sub(t1)) + blockWriteTimer.Update(t3.Sub(t2)) switch status { case CanonStatTy: log.Debug("Inserted new block", "number", block.Number(), "hash", block.Hash(), From 174083c3ae0b6217dff0ced98ec7f73e53e24b04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 13:31:42 +0200 Subject: [PATCH 06/68] eth/downloader: fix light client cht binary search issue --- eth/downloader/downloader.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f81a5cbac2..3a177ab9de 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -99,6 +99,7 @@ type Downloader struct { mode SyncMode // Synchronisation mode defining the strategy used (per sync cycle) mux *event.TypeMux // Event multiplexer to announce sync operation events + genesis uint64 // Genesis block number to limit sync to (e.g. light client CHT) queue *queue // Scheduler for selecting the hashes to download peers *peerSet // Set of active peers from which download can proceed stateDB ethdb.Database @@ -664,7 +665,28 @@ func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) } p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) if localHeight >= MaxForkAncestry { + // We're above the max reorg threshold, find the earliest fork point floor = int64(localHeight - MaxForkAncestry) + + // If we're doing a light sync, ensure the floor doesn't go below the CHT, as + // all headers before that point will be missing. + if d.mode == LightSync { + // If we dont know the current CHT position, find it + if d.genesis == 0 { + header := d.lightchain.CurrentHeader() + for header != nil { + d.genesis = header.Number.Uint64() + if floor >= int64(d.genesis)-1 { + break + } + header = d.lightchain.GetHeaderByHash(header.ParentHash) + } + } + // We already know the "genesis" block number, cap floor to that + if floor < int64(d.genesis)-1 { + floor = int64(d.genesis) - 1 + } + } } from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) From d606a7a46a62483f7fd8110de97ce91f70c4766e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 13:53:33 +0200 Subject: [PATCH 07/68] params: update CHTs for the v1.8.19 release --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 3570cb204f..ba9fac1c4a 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 203, - SectionHead: common.HexToHash("0xc9e05fc67c6a9815adc8072eb18805b53da53a9a6a273e05541e1b7542cf937a"), - CHTRoot: common.HexToHash("0xb85f42447d59f7c3e6679b9a37ed983593fd52efd6251b883592662e95769d5b"), - BloomRoot: common.HexToHash("0xf93d50cb4c49b403c6fd33cd60896d3b36184275be0a51bae4df5e8844ac624c"), + SectionIndex: 206, + SectionHead: common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"), + CHTRoot: common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"), + BloomRoot: common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 134, - SectionHead: common.HexToHash("0x17053ecbe045bebefaa01e7716cc85a4e22647e181416cc1098ccbb73a088931"), - CHTRoot: common.HexToHash("0x4d2b86422e46ed76f0e3f50f06632c409f809c8375e53c8bc0f782bcb93dd49a"), - BloomRoot: common.HexToHash("0xccba62232ee56c2967afc58f136a47ba7dc545ae586e6be666430d94516306c7"), + SectionIndex: 136, + SectionHead: common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"), + CHTRoot: common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"), + BloomRoot: common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 100, - SectionHead: common.HexToHash("0xf18f9b43e16f37b12e68818536ffe455ff18d676274ffdd856a8520ed61bb514"), - CHTRoot: common.HexToHash("0x473f5d603b1fedad75d97fd58692130b9ac9ade1aca01eb9363d79bd1c43c791"), - BloomRoot: common.HexToHash("0xa39ced3ddbb87e909c7531df2afb6414bea9c9a60ab94da9c6b467535f05326e"), + SectionIndex: 103, + SectionHead: common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"), + CHTRoot: common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"), + BloomRoot: common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From dae82f098570e15d44584f0d7f350713f4774727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 14:42:37 +0200 Subject: [PATCH 08/68] params, swarm: release Geth v1.8.19 and Swarm v0.3.7 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index b9dcc2a842..cbb23e16dd 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 19 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 19 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 17ef34f5f4..d75763fe95 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 7 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From a7501d0c419f8c1dc8b4004c115dd2d4e3b524b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 28 Nov 2018 14:44:00 +0200 Subject: [PATCH 09/68] params, swarm: start Geth v1.8.20 and Swarm v0.3.8 release cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index cbb23e16dd..2291d5f3c5 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 19 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 20 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index d75763fe95..57ac05a864 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 3dba6a6d27cc98814a23555d84cf6f016fe4635d Mon Sep 17 00:00:00 2001 From: Franklin Date: Wed, 28 Nov 2018 22:42:07 +0800 Subject: [PATCH 10/68] remove unrelated code --- core/blockchain.go | 6 +----- rpc/ipc.go | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ee1c0702b5..d173b2de29 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -445,11 +445,7 @@ func (bc *BlockChain) repair(head **types.Block) error { return nil } // Otherwise rewind one block and recheck state availability there - block := bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) - if block == nil { - return fmt.Errorf("failed to repair block, can not get block at height %d", (*head).NumberU64()) - } - (*head) = block + (*head) = bc.GetBlock((*head).ParentHash(), (*head).NumberU64()-1) } } diff --git a/rpc/ipc.go b/rpc/ipc.go index 1467f7a0c9..4cce1cf74b 100644 --- a/rpc/ipc.go +++ b/rpc/ipc.go @@ -24,7 +24,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/netutil" ) -// ServeListener accepts connections on l, serving IPC-RPC on them. +// ServeListener accepts connections on l, serving JSON-RPC on them. func (srv *Server) ServeListener(l net.Listener) error { for { conn, err := l.Accept() From 3d21d455dca6e75a635ab5742c66c80ee175c3e7 Mon Sep 17 00:00:00 2001 From: Matthew Di Ferrante Date: Thu, 29 Nov 2018 09:29:12 +0100 Subject: [PATCH 11/68] cmd/evm: commit statedb if dump is requested (#18208) Add a call `statedb.Commit(true)` if the `Dump` flag is on, as otherwise the `storage` output in the dump is always empty. --- cmd/evm/runner.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 962fc021d7..a9c8a38cae 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -206,6 +206,7 @@ func runCmd(ctx *cli.Context) error { execTime := time.Since(tstart) if ctx.GlobalBool(DumpFlag.Name) { + statedb.Commit(true) statedb.IntermediateRoot(true) fmt.Println(string(statedb.Dump())) } From 7c657fc78957d7ed302eba68bb6122b515364ac4 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Nov 2018 09:51:57 +0100 Subject: [PATCH 12/68] tests, core: update tests and make STATICCALL cause touch-delete (#18187) --- core/vm/evm.go | 6 ++++++ tests/init.go | 9 +++++++++ tests/testdata | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 968d2219ea..ba4d1e9eb8 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -339,6 +339,12 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte contract := NewContract(caller, to, new(big.Int), gas) contract.SetCallCode(&addr, evm.StateDB.GetCodeHash(addr), evm.StateDB.GetCode(addr)) + // We do an AddBalance of zero here, just in order to trigger a touch. + // This doesn't matter on Mainnet, where all empties are gone at the time of Byzantium, + // but is the correct thing to do and matters on other networks, in tests, and potential + // future scenarios + evm.StateDB.AddBalance(addr, bigZero) + // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally // when we're in Homestead this also counts for code storage gas errors. diff --git a/tests/init.go b/tests/init.go index f0a4943c13..db0457b6d7 100644 --- a/tests/init.go +++ b/tests/init.go @@ -86,6 +86,15 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), ByzantiumBlock: big.NewInt(5), }, + "ByzantiumToConstantinopleAt5": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(5), + }, } // UnsupportedForkError is returned when a test requests a fork that isn't implemented. diff --git a/tests/testdata b/tests/testdata index 95a3092038..c02a2a17c0 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 95a309203890e6244c6d4353ca411671973c13b5 +Subproject commit c02a2a17c0288a255572b37dc7ec1fcb838b9dbf From 8380a1303ce6a2fa4760a8641e47e7196a941b4c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 28 Nov 2018 20:53:02 +0100 Subject: [PATCH 13/68] vendor: update leveldb --- .../syndtr/goleveldb/leveldb/cache/cache.go | 1 - .../leveldb/comparer/bytes_comparer.go | 4 +- .../goleveldb/leveldb/comparer/comparer.go | 2 +- .../github.com/syndtr/goleveldb/leveldb/db.go | 2 +- .../syndtr/goleveldb/leveldb/db_compaction.go | 24 +++----- .../syndtr/goleveldb/leveldb/db_util.go | 2 +- .../syndtr/goleveldb/leveldb/iterator/iter.go | 4 +- .../syndtr/goleveldb/leveldb/opt/options.go | 13 +++++ .../syndtr/goleveldb/leveldb/session_util.go | 4 +- .../syndtr/goleveldb/leveldb/table.go | 30 +++++----- vendor/vendor.json | 58 +++++++++---------- 11 files changed, 76 insertions(+), 68 deletions(-) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go index c5940b232c..c36ad32359 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go @@ -331,7 +331,6 @@ func (r *Cache) delete(n *Node) bool { return deleted } } - return false } // Nodes returns number of 'cache node' in the map. diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go index 14dddf88dd..abf9fb65c7 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go @@ -29,7 +29,7 @@ func (bytesComparer) Separator(dst, a, b []byte) []byte { // Do not shorten if one string is a prefix of the other } else if c := a[i]; c < 0xff && c+1 < b[i] { dst = append(dst, a[:i+1]...) - dst[i]++ + dst[len(dst)-1]++ return dst } return nil @@ -39,7 +39,7 @@ func (bytesComparer) Successor(dst, b []byte) []byte { for i, c := range b { if c != 0xff { dst = append(dst, b[:i+1]...) - dst[i]++ + dst[len(dst)-1]++ return dst } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go index 14a28f16fc..2c522db23b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go @@ -36,7 +36,7 @@ type Comparer interface { // by any users of this package. Name() string - // Bellow are advanced functions used used to reduce the space requirements + // Bellow are advanced functions used to reduce the space requirements // for internal data structures such as index blocks. // Separator appends a sequence of bytes x to dst such that a <= x && x < b, diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go index e7ac065418..b27c38d37e 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db.go @@ -182,7 +182,7 @@ func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) { err = s.recover() if err != nil { - if !os.IsNotExist(err) || s.o.GetErrorIfMissing() { + if !os.IsNotExist(err) || s.o.GetErrorIfMissing() || s.o.GetReadOnly() { return } err = s.create() diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go index 28e50906ad..0c1b9a53b8 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go @@ -663,7 +663,7 @@ type cCmd interface { } type cAuto struct { - // Note for table compaction, an empty ackC represents it's a compaction waiting command. + // Note for table compaction, an non-empty ackC represents it's a compaction waiting command. ackC chan<- error } @@ -777,8 +777,8 @@ func (db *DB) mCompaction() { func (db *DB) tCompaction() { var ( - x cCmd - ackQ, waitQ []cCmd + x cCmd + waitQ []cCmd ) defer func() { @@ -787,10 +787,6 @@ func (db *DB) tCompaction() { panic(x) } } - for i := range ackQ { - ackQ[i].ack(ErrClosed) - ackQ[i] = nil - } for i := range waitQ { waitQ[i].ack(ErrClosed) waitQ[i] = nil @@ -821,11 +817,6 @@ func (db *DB) tCompaction() { waitQ = waitQ[:0] } } else { - for i := range ackQ { - ackQ[i].ack(nil) - ackQ[i] = nil - } - ackQ = ackQ[:0] for i := range waitQ { waitQ[i].ack(nil) waitQ[i] = nil @@ -844,9 +835,12 @@ func (db *DB) tCompaction() { switch cmd := x.(type) { case cAuto: if cmd.ackC != nil { - waitQ = append(waitQ, x) - } else { - ackQ = append(ackQ, x) + // Check the write pause state before caching it. + if db.resumeWrite() { + x.ack(nil) + } else { + waitQ = append(waitQ, x) + } } case cRange: x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go index 7ecd960d2c..3f0654894b 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go @@ -84,7 +84,7 @@ func (db *DB) checkAndCleanFiles() error { var mfds []storage.FileDesc for num, present := range tmap { if !present { - mfds = append(mfds, storage.FileDesc{storage.TypeTable, num}) + mfds = append(mfds, storage.FileDesc{Type: storage.TypeTable, Num: num}) db.logf("db@janitor table missing @%d", num) } } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go index b16e3a7045..96fb0f6859 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go @@ -40,11 +40,11 @@ type IteratorSeeker interface { Seek(key []byte) bool // Next moves the iterator to the next key/value pair. - // It returns whether the iterator is exhausted. + // It returns false if the iterator is exhausted. Next() bool // Prev moves the iterator to the previous key/value pair. - // It returns whether the iterator is exhausted. + // It returns false if the iterator is exhausted. Prev() bool } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go b/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go index 44e7d9adce..528b164233 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go @@ -158,6 +158,12 @@ type Options struct { // The default value is 8MiB. BlockCacheCapacity int + // BlockCacheEvictRemoved allows enable forced-eviction on cached block belonging + // to removed 'sorted table'. + // + // The default if false. + BlockCacheEvictRemoved bool + // BlockRestartInterval is the number of keys between restart points for // delta encoding of keys. // @@ -384,6 +390,13 @@ func (o *Options) GetBlockCacheCapacity() int { return o.BlockCacheCapacity } +func (o *Options) GetBlockCacheEvictRemoved() bool { + if o == nil { + return false + } + return o.BlockCacheEvictRemoved +} + func (o *Options) GetBlockRestartInterval() int { if o == nil || o.BlockRestartInterval <= 0 { return DefaultBlockRestartInterval diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go index 92328933cc..40cb2cf957 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go @@ -36,7 +36,7 @@ func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf func (s *session) newTemp() storage.FileDesc { num := atomic.AddInt64(&s.stTempFileNum, 1) - 1 - return storage.FileDesc{storage.TypeTemp, num} + return storage.FileDesc{Type: storage.TypeTemp, Num: num} } func (s *session) addFileRef(fd storage.FileDesc, ref int) int { @@ -190,7 +190,7 @@ func (s *session) recordCommited(rec *sessionRecord) { // Create a new manifest file; need external synchronization. func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { - fd := storage.FileDesc{storage.TypeManifest, s.allocFileNum()} + fd := storage.FileDesc{Type: storage.TypeManifest, Num: s.allocFileNum()} writer, err := s.stor.Create(fd) if err != nil { return diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table.go b/vendor/github.com/syndtr/goleveldb/leveldb/table.go index 81d18a531e..1fac60d050 100644 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table.go +++ b/vendor/github.com/syndtr/goleveldb/leveldb/table.go @@ -78,7 +78,7 @@ func newTableFile(fd storage.FileDesc, size int64, imin, imax internalKey) *tFil } func tableFileFromRecord(r atRecord) *tFile { - return newTableFile(storage.FileDesc{storage.TypeTable, r.num}, r.size, r.imin, r.imax) + return newTableFile(storage.FileDesc{Type: storage.TypeTable, Num: r.num}, r.size, r.imin, r.imax) } // tFiles hold multiple tFile. @@ -290,16 +290,17 @@ func (x *tFilesSortByNum) Less(i, j int) bool { // Table operations. type tOps struct { - s *session - noSync bool - cache *cache.Cache - bcache *cache.Cache - bpool *util.BufferPool + s *session + noSync bool + evictRemoved bool + cache *cache.Cache + bcache *cache.Cache + bpool *util.BufferPool } // Creates an empty table and returns table writer. func (t *tOps) create() (*tWriter, error) { - fd := storage.FileDesc{storage.TypeTable, t.s.allocFileNum()} + fd := storage.FileDesc{Type: storage.TypeTable, Num: t.s.allocFileNum()} fw, err := t.s.stor.Create(fd) if err != nil { return nil, err @@ -422,7 +423,7 @@ func (t *tOps) remove(f *tFile) { } else { t.s.logf("table@remove removed @%d", f.fd.Num) } - if t.bcache != nil { + if t.evictRemoved && t.bcache != nil { t.bcache.EvictNS(uint64(f.fd.Num)) } }) @@ -451,7 +452,7 @@ func newTableOps(s *session) *tOps { if !s.o.GetDisableBlockCache() { var bcacher cache.Cacher if s.o.GetBlockCacheCapacity() > 0 { - bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) + bcacher = s.o.GetBlockCacher().New(s.o.GetBlockCacheCapacity()) } bcache = cache.NewCache(bcacher) } @@ -459,11 +460,12 @@ func newTableOps(s *session) *tOps { bpool = util.NewBufferPool(s.o.GetBlockSize() + 5) } return &tOps{ - s: s, - noSync: s.o.GetNoSync(), - cache: cache.NewCache(cacher), - bcache: bcache, - bpool: bpool, + s: s, + noSync: s.o.GetNoSync(), + evictRemoved: s.o.GetBlockCacheEvictRemoved(), + cache: cache.NewCache(cacher), + bcache: bcache, + bpool: bpool, } } diff --git a/vendor/vendor.json b/vendor/vendor.json index 1bfe09da72..a4fee11aa5 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -455,76 +455,76 @@ "revisionTime": "2017-07-05T02:17:15Z" }, { - "checksumSHA1": "k6zbR5hiI10hkWtiK91rIY5s5/E=", + "checksumSHA1": "LV0VMVON7xY1ttV+s2ph83ntmDQ=", "path": "github.com/syndtr/goleveldb/leveldb", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "EKIow7XkgNdWvR/982ffIZxKG8Y=", + "checksumSHA1": "mPNraL2edpk/2FYq26rSXfMHbJg=", "path": "github.com/syndtr/goleveldb/leveldb/cache", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "5KPgnvCPlR0ysDAqo6jApzRQ3tw=", + "checksumSHA1": "UA+PKDKWlDnE2OZblh23W6wZwbY=", "path": "github.com/syndtr/goleveldb/leveldb/comparer", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "1DRAxdlWzS4U0xKN/yQ/fdNN7f0=", "path": "github.com/syndtr/goleveldb/leveldb/errors", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "eqKeD6DS7eNCtxVYZEHHRKkyZrw=", "path": "github.com/syndtr/goleveldb/leveldb/filter", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "weSsccMav4BCerDpSLzh3mMxAYo=", + "checksumSHA1": "hPyFsMiqZ1OB7MX+6wIAA6nsdtc=", "path": "github.com/syndtr/goleveldb/leveldb/iterator", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "gJY7bRpELtO0PJpZXgPQ2BYFJ88=", "path": "github.com/syndtr/goleveldb/leveldb/journal", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "MtYY1b2234y/MlS+djL8tXVAcQs=", "path": "github.com/syndtr/goleveldb/leveldb/memdb", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { - "checksumSHA1": "UmQeotV+m8/FduKEfLOhjdp18rs=", + "checksumSHA1": "o2TorI3z+vc+EBMJ8XeFoUmXBtU=", "path": "github.com/syndtr/goleveldb/leveldb/opt", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "ZnyuciM+R19NG8L5YS3TIJdo1e8=", "path": "github.com/syndtr/goleveldb/leveldb/storage", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "gWFPMz8OQeul0t54RM66yMTX49g=", "path": "github.com/syndtr/goleveldb/leveldb/table", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "V/Dh7NV0/fy/5jX1KaAjmGcNbzI=", "path": "github.com/syndtr/goleveldb/leveldb/util", - "revision": "c4c61651e9e37fa117f53c5a906d3b63090d8445", - "revisionTime": "2018-07-08T03:05:51Z" + "revision": "b001fa50d6b27f3f0bb175a87d0cb55426d0a0ae", + "revisionTime": "2018-11-28T10:09:59Z" }, { "checksumSHA1": "nD6S4KB0S+YHxVMDDE+w3PyXaMk=", From 55a4ff806fc1c8018055c56abcc6857502447902 Mon Sep 17 00:00:00 2001 From: zah Date: Thu, 29 Nov 2018 11:57:00 +0200 Subject: [PATCH 14/68] remove a no-op line in the code (#17760) --- les/flowcontrol/control.go | 1 - 1 file changed, 1 deletion(-) diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index d50eb809cc..8ef4ba511f 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -82,7 +82,6 @@ func (peer *ClientNode) RequestProcessed(cost uint64) (bv, realCost uint64) { time := mclock.Now() peer.recalcBV(time) peer.bufValue -= cost - peer.recalcBV(time) rcValue, rcost := peer.cm.processed(peer.cmNode, time) if rcValue < peer.params.BufLimit { bv := peer.params.BufLimit - rcValue From a4428c505e84aaa1982d4cc522e0c0ffe608890a Mon Sep 17 00:00:00 2001 From: Eugene Valeyev Date: Thu, 29 Nov 2018 13:02:31 +0300 Subject: [PATCH 15/68] mobile: added constructor for BigInts (#17828) --- mobile/big.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mobile/big.go b/mobile/big.go index dd7b158786..86ea93245a 100644 --- a/mobile/big.go +++ b/mobile/big.go @@ -84,6 +84,13 @@ func (bi *BigInt) SetString(x string, base int) { // BigInts represents a slice of big ints. type BigInts struct{ bigints []*big.Int } +// NewBigInts creates a slice of uninitialized big numbers. +func NewBigInts(size int) *BigInts { + return &BigInts{ + bigints: make([]*big.Int, size), + } +} + // Size returns the number of big ints in the slice. func (bi *BigInts) Size() int { return len(bi.bigints) From 32d35c9c088463efac49aeb0f3e6d48cfb373a40 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Thu, 29 Nov 2018 18:04:56 +0800 Subject: [PATCH 16/68] accounts/keystore: delete the redundant keystore in filename (#17930) * accounts/keystore: reduce file name length * accounts/keystore: reduce code line width --- accounts/keystore/account_cache.go | 5 ++++- accounts/keystore/key.go | 8 ++++++-- .../keystore/{keystore_passphrase.go => passphrase.go} | 1 + .../{keystore_passphrase_test.go => passphrase_test.go} | 0 accounts/keystore/{keystore_plain.go => plain.go} | 0 .../keystore/{keystore_plain_test.go => plain_test.go} | 0 accounts/keystore/presale.go | 8 +++++++- accounts/keystore/{keystore_wallet.go => wallet.go} | 0 8 files changed, 18 insertions(+), 4 deletions(-) rename accounts/keystore/{keystore_passphrase.go => passphrase.go} (99%) rename accounts/keystore/{keystore_passphrase_test.go => passphrase_test.go} (100%) rename accounts/keystore/{keystore_plain.go => plain.go} (100%) rename accounts/keystore/{keystore_plain_test.go => plain_test.go} (100%) rename accounts/keystore/{keystore_wallet.go => wallet.go} (100%) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index da3a46eb80..8f660e282f 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -265,7 +265,10 @@ func (ac *accountCache) scanAccounts() error { case (addr == common.Address{}): log.Debug("Failed to decode keystore key", "path", path, "err", "missing or zero address") default: - return &accounts.Account{Address: addr, URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}} + return &accounts.Account{ + Address: addr, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: path}, + } } return nil } diff --git a/accounts/keystore/key.go b/accounts/keystore/key.go index 0564751c43..84d8df0c5a 100644 --- a/accounts/keystore/key.go +++ b/accounts/keystore/key.go @@ -171,7 +171,10 @@ func storeNewKey(ks keyStore, rand io.Reader, auth string) (*Key, accounts.Accou if err != nil { return nil, accounts.Account{}, err } - a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}} + a := accounts.Account{ + Address: key.Address, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(keyFileName(key.Address))}, + } if err := ks.StoreKey(a.URL.Path, key, auth); err != nil { zeroKey(key.PrivateKey) return nil, a, err @@ -224,5 +227,6 @@ func toISO8601(t time.Time) string { } else { tz = fmt.Sprintf("%03d00", offset/3600) } - return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz) + return fmt.Sprintf("%04d-%02d-%02dT%02d-%02d-%02d.%09d%s", + t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second(), t.Nanosecond(), tz) } diff --git a/accounts/keystore/keystore_passphrase.go b/accounts/keystore/passphrase.go similarity index 99% rename from accounts/keystore/keystore_passphrase.go rename to accounts/keystore/passphrase.go index 9794f32fe1..a0b6cf5385 100644 --- a/accounts/keystore/keystore_passphrase.go +++ b/accounts/keystore/passphrase.go @@ -233,6 +233,7 @@ func DecryptKey(keyjson []byte, auth string) (*Key, error) { PrivateKey: key, }, nil } + func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) { if cryptoJson.Cipher != "aes-128-ctr" { return nil, fmt.Errorf("Cipher not supported: %v", cryptoJson.Cipher) diff --git a/accounts/keystore/keystore_passphrase_test.go b/accounts/keystore/passphrase_test.go similarity index 100% rename from accounts/keystore/keystore_passphrase_test.go rename to accounts/keystore/passphrase_test.go diff --git a/accounts/keystore/keystore_plain.go b/accounts/keystore/plain.go similarity index 100% rename from accounts/keystore/keystore_plain.go rename to accounts/keystore/plain.go diff --git a/accounts/keystore/keystore_plain_test.go b/accounts/keystore/plain_test.go similarity index 100% rename from accounts/keystore/keystore_plain_test.go rename to accounts/keystore/plain_test.go diff --git a/accounts/keystore/presale.go b/accounts/keystore/presale.go index 1554294e14..03055245f5 100644 --- a/accounts/keystore/presale.go +++ b/accounts/keystore/presale.go @@ -38,7 +38,13 @@ func importPreSaleKey(keyStore keyStore, keyJSON []byte, password string) (accou return accounts.Account{}, nil, err } key.Id = uuid.NewRandom() - a := accounts.Account{Address: key.Address, URL: accounts.URL{Scheme: KeyStoreScheme, Path: keyStore.JoinPath(keyFileName(key.Address))}} + a := accounts.Account{ + Address: key.Address, + URL: accounts.URL{ + Scheme: KeyStoreScheme, + Path: keyStore.JoinPath(keyFileName(key.Address)), + }, + } err = keyStore.StoreKey(a.URL.Path, key, password) return a, key, err } diff --git a/accounts/keystore/keystore_wallet.go b/accounts/keystore/wallet.go similarity index 100% rename from accounts/keystore/keystore_wallet.go rename to accounts/keystore/wallet.go From cf62bd2e8878d880aa9d5b20c05bb348111f6298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 29 Nov 2018 12:47:29 +0200 Subject: [PATCH 17/68] cmd/utils: max out the OS file allowance, don't cap to 2K --- cmd/utils/flags.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d0597c2f18..6a285fcb39 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -824,17 +824,12 @@ func setIPC(ctx *cli.Context, cfg *node.Config) { // makeDatabaseHandles raises out the number of allowed file handles per process // for Geth and returns half of the allowance to assign to the database. func makeDatabaseHandles() int { - limit, err := fdlimit.Current() + limit, err := fdlimit.Maximum() if err != nil { Fatalf("Failed to retrieve file descriptor allowance: %v", err) } - if limit < 2048 { - if err := fdlimit.Raise(2048); err != nil { - Fatalf("Failed to raise file descriptor allowance: %v", err) - } - } - if limit > 2048 { // cap database file descriptors even if more is available - limit = 2048 + if err := fdlimit.Raise(uint64(limit)); err != nil { + Fatalf("Failed to raise file descriptor allowance: %v", err) } return limit / 2 // Leave half for networking and other stuff } From 7e7781ffaa5c5486c5e7b66dea4986e063524f63 Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Thu, 29 Nov 2018 17:43:15 +0100 Subject: [PATCH 18/68] cmd/swarm: add flag for application name (swarm or swarm-private) (#18189) * cmd/swarm: add flag for application name (swarm or swarm-private) * cmd/swarm/swarm-smoke: return correct exit code * cmd/swarm/swarm-smoke: remove colorable * remove swarm/grafana_dashboards --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 2 +- cmd/swarm/swarm-smoke/main.go | 8 + cmd/swarm/swarm-smoke/upload_and_sync.go | 11 +- swarm/grafana_dashboards/ldbstore.json | 2278 ------------ swarm/grafana_dashboards/swarm.json | 3198 ----------------- 5 files changed, 14 insertions(+), 5483 deletions(-) delete mode 100644 swarm/grafana_dashboards/ldbstore.json delete mode 100644 swarm/grafana_dashboards/swarm.json diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 0328d656f3..7ec1528263 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -33,7 +33,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) - generateEndpoints(scheme, cluster, from, to) + generateEndpoints(scheme, cluster, appName, from, to) log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing") diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 4ff17fd5b7..845998dc12 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -29,6 +29,7 @@ var ( endpoints []string includeLocalhost bool cluster string + appName string scheme string filesize int from int @@ -49,6 +50,12 @@ func main() { Usage: "cluster to point to (prod or a given namespace)", Destination: &cluster, }, + cli.StringFlag{ + Name: "app", + Value: "swarm", + Usage: "application to point to (swarm or swarm-private)", + Destination: &appName, + }, cli.IntFlag{ Name: "cluster-from", Value: 8501, @@ -107,5 +114,6 @@ func main() { err := app.Run(os.Args) if err != nil { log.Error(err.Error()) + os.Exit(1) } } diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 7872421d3d..3843457dc4 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -33,20 +33,19 @@ import ( "time" "github.com/ethereum/go-ethereum/log" - colorable "github.com/mattn/go-colorable" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) -func generateEndpoints(scheme string, cluster string, from int, to int) { +func generateEndpoints(scheme string, cluster string, app string, from int, to int) { if cluster == "prod" { for port := from; port <= to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } } else { for port := from; port <= to; port++ { - endpoints = append(endpoints, fmt.Sprintf("%s://swarm-%v-%s.stg.swarm-gateways.net", scheme, port, cluster)) + endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) } } @@ -57,11 +56,11 @@ func generateEndpoints(scheme string, cluster string, from int, to int) { func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) + defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) - generateEndpoints(scheme, cluster, from, to) + generateEndpoints(scheme, cluster, appName, from, to) log.Info("uploading to " + endpoints[0] + " and syncing") diff --git a/swarm/grafana_dashboards/ldbstore.json b/swarm/grafana_dashboards/ldbstore.json deleted file mode 100644 index 2d64380ba8..0000000000 --- a/swarm/grafana_dashboards/ldbstore.json +++ /dev/null @@ -1,2278 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:325", - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": 5, - "iteration": 1527598894689, - "links": [], - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 40, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 42, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.cachehit.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get cachehit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 43, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.cachemiss.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get cachemiss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 44, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total LocalStore.GetOrCreateRequest", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 47, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.errfetching.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore GetOrCreateRequest ErrFetching", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 45, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.hit.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore.GetOrCreateRequest hit", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 49, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.getorcreaterequest.miss.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore GetOrCreateRequest miss", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 48, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.error.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get error", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 19 - }, - "id": 46, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.localstore.get.errfetching.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LocalStore get ErrFetching", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LocalStore", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 27, - "panels": [], - "title": "LDBStore", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 29, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.get.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 30, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.put.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore put", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 31, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.synciterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore SyncIterator", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 32, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.synciterator.seek.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore SyncIterator Seek/Next", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 50, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.collectgarbage.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore Collect Garbage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 51, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbstore.collectgarbage.delete.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBStore Collect Garbage - Actual Deletes", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 34, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.get.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.write.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase write", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 45 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.ldbdatabase.newiterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LDBDatabase NewIterator", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LDBDatabase", - "type": "row" - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "10s", - "value": "10s" - }, - "hide": 0, - "label": "resolution", - "name": "myinterval", - "options": [ - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "100s", - "value": "100s" - } - ], - "query": "5s,10s,30s,100s", - "refresh": 2, - "type": "interval" - }, - { - "allValue": null, - "current": { - "text": "swarm_30399 + swarm_30400 + swarm_30401", - "value": [ - "swarm_30399", - "swarm_30400", - "swarm_30401" - ] - }, - "datasource": "metrics", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "host", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"host\"", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "swarm.http.request.GET.time.span", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "LDBStore and LDBDatabase", - "uid": "zS6beG7iz", - "version": 28 -} diff --git a/swarm/grafana_dashboards/swarm.json b/swarm/grafana_dashboards/swarm.json deleted file mode 100644 index 3ee244d154..0000000000 --- a/swarm/grafana_dashboards/swarm.json +++ /dev/null @@ -1,3198 +0,0 @@ -{ - "annotations": { - "list": [ - { - "$$hashKey": "object:147", - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": 2, - "iteration": 1527598859072, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 34, - "panels": [], - "title": "P2P", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 36, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.send.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P Send() - messages sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 37, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "p95($tag_host)", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.send_t.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P Send() timer - 95%ile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 38, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "1 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.1.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - }, - { - "alias": "2 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.2.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - }, - { - "alias": "3 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority.3.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "C", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P SendPriority() - messages sent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 39, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "1 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority_t.1.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - }, - { - "alias": "2 $tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.sendpriority_t.2.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "P2P SendPriority() timer - 95%ile", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 19 - }, - "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$__interval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "none" - ], - "type": "fill" - } - ], - "measurement": "swarm.registry.peers.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "last" - } - ] - ], - "tags": [] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Registry Peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 32, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.stack.uptime.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Uptime", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "Uptime", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 29 - }, - "id": 28, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "GET", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - }, - { - "alias": "POST", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "null" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.POST.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "B", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Total HTTP Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 26, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP GET requests 95% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.GET.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p50" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP GET requests 50% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 13 - }, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "POST", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.http.request.POST.time.span", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "p95" - ], - "type": "field" - }, - { - "params": [], - "type": "mean" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "HTTP POST requests 95% timer", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ns", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "HTTP", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 30 - }, - "id": 30, - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader read() calls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 18, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.err.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader read errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 17, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.lazychunkreader.read.bytes.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "LazyChunkReader bytes read", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "title": "LazyChunkReader", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 25, - "panels": [], - "title": "All measurements", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.api.get.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "API Get (BZZ)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.request_from_peers.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request from peers", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 38 - }, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.received_chunks.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Received chunks", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.storage.cache.requests.size.gauge", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "max" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Requests cache entries", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.network.stream.handle_retrieve_request_msg.count.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handle retrieve request msg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 20, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.syncer.setnextbatch.iterator.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "syncer setnextbatch iterator calls", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 50 - }, - "id": 21, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlewantedhashesmsg.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleWantedHashesMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 50 - }, - "id": 22, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlesubscribemsg.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleSubscribeMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 23, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handlewantedhashesmsg.actualget.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer HandleWantedHashesMsg actual get", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "metrics", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 19, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "alias": "$tag_host", - "dsType": "influxdb", - "groupBy": [ - { - "params": [ - "$myinterval" - ], - "type": "time" - }, - { - "params": [ - "host" - ], - "type": "tag" - }, - { - "params": [ - "0" - ], - "type": "fill" - } - ], - "measurement": "swarm.peer.handleofferedhashes.count", - "orderByTime": "ASC", - "policy": "default", - "refId": "A", - "resultFormat": "time_series", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "field" - }, - { - "params": [], - "type": "sum" - } - ] - ], - "tags": [ - { - "key": "host", - "operator": "=~", - "value": "/^$host$/" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "peer OfferedHashesMsg", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "30s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "text": "10s", - "value": "10s" - }, - "hide": 0, - "label": "resolution", - "name": "myinterval", - "options": [ - { - "selected": false, - "text": "5s", - "value": "5s" - }, - { - "selected": true, - "text": "10s", - "value": "10s" - }, - { - "selected": false, - "text": "30s", - "value": "30s" - }, - { - "selected": false, - "text": "100s", - "value": "100s" - } - ], - "query": "5s,10s,30s,100s", - "refresh": 2, - "type": "interval" - }, - { - "allValue": null, - "current": { - "text": "swarm_30399 + swarm_30400 + swarm_30401 + swarm_30402", - "value": [ - "swarm_30399", - "swarm_30400", - "swarm_30401", - "swarm_30402" - ] - }, - "datasource": "metrics", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "host", - "options": [], - "query": "SHOW TAG VALUES WITH KEY = \"host\"", - "refresh": 1, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "swarm.http.request.GET.time.span", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-15m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Swarm", - "uid": "vmEtxxgmz", - "version": 138 -} From ef8ced4151006c84e77ab8db88cf6fc1bb09a716 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 30 Nov 2018 10:22:53 +0100 Subject: [PATCH 19/68] vendor: update github.com/karalabe/hid (#18213) Fixes #15101 because hidapi is no longer being called from an init function. --- vendor/github.com/karalabe/hid/appveyor.yml | 4 ++-- vendor/github.com/karalabe/hid/hid_disabled.go | 2 +- vendor/github.com/karalabe/hid/hid_enabled.go | 12 ++++++------ vendor/vendor.json | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/vendor/github.com/karalabe/hid/appveyor.yml b/vendor/github.com/karalabe/hid/appveyor.yml index f43958747c..84b3c95ff7 100644 --- a/vendor/github.com/karalabe/hid/appveyor.yml +++ b/vendor/github.com/karalabe/hid/appveyor.yml @@ -22,8 +22,8 @@ environment: install: - rmdir C:\go /s /q - - appveyor DownloadFile https://storage.googleapis.com/golang/go1.8.windows-%GOARCH%.zip - - 7z x go1.8.windows-%GOARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://storage.googleapis.com/golang/go1.10.1.windows-%GOARCH%.zip + - 7z x go1.10.1.windows-%GOARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/vendor/github.com/karalabe/hid/hid_disabled.go b/vendor/github.com/karalabe/hid/hid_disabled.go index 1f40263791..0f266ba58a 100644 --- a/vendor/github.com/karalabe/hid/hid_disabled.go +++ b/vendor/github.com/karalabe/hid/hid_disabled.go @@ -36,7 +36,7 @@ func (info DeviceInfo) Open() (*Device, error) { // Close releases the HID USB device handle. On platforms that this file implements // the method is just a noop. -func (dev *Device) Close() {} +func (dev *Device) Close() error { return nil } // Write sends an output report to a HID device. On platforms that this file // implements the method just returns an error. diff --git a/vendor/github.com/karalabe/hid/hid_enabled.go b/vendor/github.com/karalabe/hid/hid_enabled.go index 419273be6a..e95e5792d9 100644 --- a/vendor/github.com/karalabe/hid/hid_enabled.go +++ b/vendor/github.com/karalabe/hid/hid_enabled.go @@ -41,6 +41,7 @@ package hid #endif */ import "C" + import ( "errors" "runtime" @@ -57,11 +58,6 @@ import ( // > "subsequent calls will cause the hid manager to release previously enumerated devices" var enumerateLock sync.Mutex -func init() { - // Initialize the HIDAPI library - C.hid_init() -} - // Supported returns whether this platform is supported by the HID library or not. // The goal of this method is to allow programatically handling platforms that do // not support USB HID and not having to fall back to build constraints. @@ -113,6 +109,9 @@ func Enumerate(vendorID uint16, productID uint16) []DeviceInfo { // Open connects to an HID device by its path name. func (info DeviceInfo) Open() (*Device, error) { + enumerateLock.Lock() + defer enumerateLock.Unlock() + path := C.CString(info.Path) defer C.free(unsafe.Pointer(path)) @@ -135,7 +134,7 @@ type Device struct { } // Close releases the HID USB device handle. -func (dev *Device) Close() { +func (dev *Device) Close() error { dev.lock.Lock() defer dev.lock.Unlock() @@ -143,6 +142,7 @@ func (dev *Device) Close() { C.hid_close(dev.device) dev.device = nil } + return nil } // Write sends an output report to a HID device. diff --git a/vendor/vendor.json b/vendor/vendor.json index a4fee11aa5..883d02097d 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -267,10 +267,10 @@ "revisionTime": "2017-04-30T22:20:11Z" }, { - "checksumSHA1": "UpjhOUZ1+0zNt+iIvdtECSHXmTs=", + "checksumSHA1": "6XsjAARQFvlW6dS15al0ibTFPOQ=", "path": "github.com/karalabe/hid", - "revision": "f00545f9f3748e591590be3732d913c77525b10f", - "revisionTime": "2017-08-21T10:38:37Z", + "revision": "d815e0c1a2e2082a287a2806bc90bc8fc7b276a9", + "revisionTime": "2018-11-28T19:21:57Z", "tree": true }, { From 54abb97e3be4fb2d6f557758aaa3a8291b0979fe Mon Sep 17 00:00:00 2001 From: needkane <604476380@qq.com> Date: Sat, 1 Dec 2018 05:38:37 +0800 Subject: [PATCH 20/68] p2p: use errors.New instead of fmt.Errorf (#18193) --- p2p/server.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/p2p/server.go b/p2p/server.go index 6678608634..566f01ffc5 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -22,7 +22,6 @@ import ( "crypto/ecdsa" "encoding/hex" "errors" - "fmt" "net" "sort" "sync" @@ -391,7 +390,7 @@ type sharedUDPConn struct { func (s *sharedUDPConn) ReadFromUDP(b []byte) (n int, addr *net.UDPAddr, err error) { packet, ok := <-s.unhandled if !ok { - return 0, nil, fmt.Errorf("Connection was closed") + return 0, nil, errors.New("Connection was closed") } l := len(packet.Data) if l > len(b) { @@ -425,7 +424,7 @@ func (srv *Server) Start() (err error) { // static fields if srv.PrivateKey == nil { - return fmt.Errorf("Server.PrivateKey must be set to a non-nil key") + return errors.New("Server.PrivateKey must be set to a non-nil key") } if srv.newTransport == nil { srv.newTransport = newRLPX @@ -903,7 +902,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro if dialDest != nil { dialPubkey = new(ecdsa.PublicKey) if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { - return fmt.Errorf("dial destination doesn't have a secp256k1 public key") + return errors.New("dial destination doesn't have a secp256k1 public key") } } // Run the encryption handshake. @@ -937,7 +936,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro return err } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { - clog.Trace("Wrong devp2p handshake identity", "phsid", fmt.Sprintf("%x", phs.ID)) + clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) return DiscUnexpectedIdentity } c.caps, c.name = phs.Caps, phs.Name From 085f89172f0af9d9685d228ad163d639041f1217 Mon Sep 17 00:00:00 2001 From: lash Date: Sat, 1 Dec 2018 07:07:18 +0100 Subject: [PATCH 21/68] swarm/pss: Add same api interface for all Send* methods (#18218) --- swarm/pss/api.go | 4 ++++ swarm/pss/pss_test.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/swarm/pss/api.go b/swarm/pss/api.go index dd55b2a702..587382d729 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -164,6 +164,10 @@ func (pssapi *API) SendSym(symkeyhex string, topic Topic, msg hexutil.Bytes) err return pssapi.Pss.SendSym(symkeyhex, topic, msg[:]) } +func (pssapi *API) SendRaw(addr hexutil.Bytes, topic Topic, msg hexutil.Bytes) error { + return pssapi.Pss.SendRaw(PssAddress(addr), topic, msg[:]) +} + func (pssapi *API) GetPeerTopics(pubkeyhex string) ([]Topic, error) { topics, _, err := pssapi.Pss.GetPublickeyPeers(pubkeyhex) return topics, err diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 32404aaaf9..72f62acd94 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -1064,7 +1064,7 @@ func testSendRaw(t *testing.T) { // send and verify delivery lmsg := []byte("plugh") - err = clients[1].Call(nil, "pss_sendRaw", loaddrhex, topic, lmsg) + err = clients[1].Call(nil, "pss_sendRaw", loaddrhex, topic, hexutil.Encode(lmsg)) if err != nil { t.Fatal(err) } @@ -1077,7 +1077,7 @@ func testSendRaw(t *testing.T) { t.Fatalf("test message (left) timed out: %v", cerr) } rmsg := []byte("xyzzy") - err = clients[0].Call(nil, "pss_sendRaw", roaddrhex, topic, rmsg) + err = clients[0].Call(nil, "pss_sendRaw", roaddrhex, topic, hexutil.Encode(rmsg)) if err != nil { t.Fatal(err) } From efe5886877d3f171fae5adfd1e5f1c585e16946b Mon Sep 17 00:00:00 2001 From: vim88 Date: Sun, 2 Dec 2018 14:00:42 +0200 Subject: [PATCH 22/68] signer/core: Fixes typo of method name in comment. (#18222) --- signer/core/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/core/api.go b/signer/core/api.go index 2b96cdb5f3..e9a3357855 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -82,7 +82,7 @@ type SignerUI interface { // OnSignerStartup is invoked when the signer boots, and tells the UI info about external API location and version // information OnSignerStartup(info StartupInfo) - // OnInputRequried is invoked when clef requires user input, for example master password or + // OnInputRequired is invoked when clef requires user input, for example master password or // pin-code for unlocking hardware wallets OnInputRequired(info UserInputRequest) (UserInputResponse, error) } From f850123ec7500af732b73c2501f1ef36d80ee01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Sun, 2 Dec 2018 13:03:31 +0100 Subject: [PATCH 23/68] Changed http:// to https:// on JSON-RPC link (#18224) Changed http:// to https:// on JSON-RPC link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4b62bfde03..7593dd0905 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ HTTP based JSON-RPC API options: * `--ipcpath` Filename for IPC socket/pipe within the datadir (explicit paths escape it) You'll need to use your own programming environments' capabilities (libraries, tools, etc) to connect -via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](http://www.jsonrpc.org/specification) +via HTTP, WS or IPC to a Geth node configured with the above flags and you'll need to speak [JSON-RPC](https://www.jsonrpc.org/specification) on all transports. You can reuse the same connection for multiple requests! **Note: Please understand the security implications of opening up an HTTP/WS based transport before From 4825d9c3dd7d07eadfda8a6d5c18efb67df67088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 Dec 2018 12:17:08 +0200 Subject: [PATCH 24/68] cmd/puppeth: enforce lowercase network names --- cmd/puppeth/puppeth.go | 4 ++-- cmd/puppeth/wizard_intro.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index f9b8fe4811..df3fff51b0 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -49,8 +49,8 @@ func main() { rand.Seed(time.Now().UnixNano()) network := c.String("network") - if strings.Contains(network, " ") || strings.Contains(network, "-") { - log.Crit("No spaces or hyphens allowed in network name") + if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { + log.Crit("No spaces, hyphens or capital letters allowed in network name") } // Start the wizard and relinquish control makeWizard(c.String("network")).run() diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 60aa0f7ffb..3db9a1087e 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -61,14 +61,14 @@ func (w *wizard) run() { // Make sure we have a good network name to work with fmt.Println() // Docker accepts hyphens in image names, but doesn't like it for container names if w.network == "" { - fmt.Println("Please specify a network name to administer (no spaces or hyphens, please)") + fmt.Println("Please specify a network name to administer (no spaces, hyphens or capital letters please)") for { w.network = w.readString() - if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") { + if !strings.Contains(w.network, " ") && !strings.Contains(w.network, "-") && strings.ToLower(w.network) == w.network { fmt.Printf("\nSweet, you can set this via --network=%s next time!\n\n", w.network) break } - log.Error("I also like to live dangerously, still no spaces or hyphens") + log.Error("I also like to live dangerously, still no spaces, hyphens or capital letters") } } log.Info("Administering Ethereum network", "name", w.network) From 8698fbabf6c7811e8eec6e84512510e9c9a4eb45 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 24 Nov 2018 23:22:25 +0100 Subject: [PATCH 25/68] cmd/puppeth: implement chainspec converters --- cmd/puppeth/genesis.go | 388 ++++++++++++++--------- cmd/puppeth/module_dashboard.go | 2 +- cmd/puppeth/puppeth.go | 32 ++ cmd/puppeth/puppeth_test.go | 91 ++++++ cmd/puppeth/testdata/stureby_aleth.json | 112 +++++++ cmd/puppeth/testdata/stureby_geth.json | 47 +++ cmd/puppeth/testdata/stureby_parity.json | 181 +++++++++++ cmd/puppeth/wizard.go | 22 ++ cmd/puppeth/wizard_genesis.go | 67 +++- 9 files changed, 780 insertions(+), 162 deletions(-) create mode 100644 cmd/puppeth/puppeth_test.go create mode 100644 cmd/puppeth/testdata/stureby_aleth.json create mode 100644 cmd/puppeth/testdata/stureby_geth.json create mode 100644 cmd/puppeth/testdata/stureby_parity.json diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 5f39a889d1..83212a126b 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -20,35 +20,41 @@ import ( "encoding/binary" "errors" "math" + "math/big" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + math2 "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/params" ) -// cppEthereumGenesisSpec represents the genesis specification format used by the +// alethGenesisSpec represents the genesis specification format used by the // C++ Ethereum implementation. -type cppEthereumGenesisSpec struct { +type alethGenesisSpec struct { SealEngine string `json:"sealEngine"` Params struct { - AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` - HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` - EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` - EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` - ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` - ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` - NetworkID hexutil.Uint64 `json:"networkID"` - ChainID hexutil.Uint64 `json:"chainID"` - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` + AccountStartNonce math2.HexOrDecimal64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + HomesteadForkBlock hexutil.Uint64 `json:"homesteadForkBlock"` + DaoHardforkBlock math2.HexOrDecimal64 `json:"daoHardforkBlock"` + EIP150ForkBlock hexutil.Uint64 `json:"EIP150ForkBlock"` + EIP158ForkBlock hexutil.Uint64 `json:"EIP158ForkBlock"` + ByzantiumForkBlock hexutil.Uint64 `json:"byzantiumForkBlock"` + ConstantinopleForkBlock hexutil.Uint64 `json:"constantinopleForkBlock"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + MaxGasLimit hexutil.Uint64 `json:"maxGasLimit"` + TieBreakingGas bool `json:"tieBreakingGas"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *math2.HexOrDecimal256 `json:"difficultyBoundDivisor"` + DurationLimit *math2.HexOrDecimal256 `json:"durationLimit"` + BlockReward *hexutil.Big `json:"blockReward"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + AllowFutureBlocks bool `json:"allowFutureBlocks""` } `json:"params"` Genesis struct { @@ -62,57 +68,68 @@ type cppEthereumGenesisSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Accounts map[common.Address]*cppEthereumGenesisSpecAccount `json:"accounts"` + Accounts map[common.UnprefixedAddress]*alethGenesisSpecAccount `json:"accounts"` } -// cppEthereumGenesisSpecAccount is the prefunded genesis account and/or precompiled +// alethGenesisSpecAccount is the prefunded genesis account and/or precompiled // contract definition. -type cppEthereumGenesisSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` - Precompiled *cppEthereumGenesisSpecBuiltin `json:"precompiled,omitempty"` +type alethGenesisSpecAccount struct { + Balance *math2.HexOrDecimal256 `json:"balance"` + Nonce uint64 `json:"nonce,omitempty"` + Precompiled *alethGenesisSpecBuiltin `json:"precompiled,omitempty"` } -// cppEthereumGenesisSpecBuiltin is the precompiled contract definition. -type cppEthereumGenesisSpecBuiltin struct { - Name string `json:"name,omitempty"` - StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` - Linear *cppEthereumGenesisSpecLinearPricing `json:"linear,omitempty"` +// alethGenesisSpecBuiltin is the precompiled contract definition. +type alethGenesisSpecBuiltin struct { + Name string `json:"name,omitempty"` + StartingBlock hexutil.Uint64 `json:"startingBlock,omitempty"` + Linear *alethGenesisSpecLinearPricing `json:"linear,omitempty"` } -type cppEthereumGenesisSpecLinearPricing struct { +type alethGenesisSpecLinearPricing struct { Base uint64 `json:"base"` Word uint64 `json:"word"` } -// newCppEthereumGenesisSpec converts a go-ethereum genesis block into a Parity specific +// newAlethGenesisSpec converts a go-ethereum genesis block into a Aleth-specific // chain specification format. -func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEthereumGenesisSpec, error) { - // Only ethash is currently supported between go-ethereum and cpp-ethereum +func newAlethGenesisSpec(network string, genesis *core.Genesis) (*alethGenesisSpec, error) { + // Only ethash is currently supported between go-ethereum and aleth if genesis.Config.Ethash == nil { return nil, errors.New("unsupported consensus engine") } - // Reconstruct the chain spec in Parity's format - spec := &cppEthereumGenesisSpec{ + // Reconstruct the chain spec in Aleth format + spec := &alethGenesisSpec{ SealEngine: "Ethash", } + // Some defaults spec.Params.AccountStartNonce = 0 + spec.Params.TieBreakingGas = false + spec.Params.AllowFutureBlocks = false + spec.Params.DaoHardforkBlock = 0 + spec.Params.HomesteadForkBlock = (hexutil.Uint64)(genesis.Config.HomesteadBlock.Uint64()) spec.Params.EIP150ForkBlock = (hexutil.Uint64)(genesis.Config.EIP150Block.Uint64()) spec.Params.EIP158ForkBlock = (hexutil.Uint64)(genesis.Config.EIP158Block.Uint64()) - spec.Params.ByzantiumForkBlock = (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()) - spec.Params.ConstantinopleForkBlock = (hexutil.Uint64)(math.MaxUint64) + + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) - spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxUint64) + spec.Params.MaxGasLimit = (hexutil.Uint64)(math.MaxInt64) spec.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) - spec.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) - spec.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) + spec.Params.DifficultyBoundDivisor = (*math2.HexOrDecimal256)(params.DifficultyBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) + spec.Params.DurationLimit = (*math2.HexOrDecimal256)(params.DurationLimit) spec.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) spec.Genesis.Nonce = (hexutil.Bytes)(make([]byte, 8)) @@ -126,77 +143,104 @@ func newCppEthereumGenesisSpec(network string, genesis *core.Genesis) (*cppEther spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*cppEthereumGenesisSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &cppEthereumGenesisSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, - } - } - spec.Accounts[common.BytesToAddress([]byte{1})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ecrecover", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 3000}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "sha256", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 60, Word: 12}, - } - spec.Accounts[common.BytesToAddress([]byte{3})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "ripemd160", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 600, Word: 120}, - } - spec.Accounts[common.BytesToAddress([]byte{4})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "identity", Linear: &cppEthereumGenesisSpecLinearPricing{Base: 15, Word: 3}, + spec.setAccount(address, account) } + + spec.setPrecompile(1, &alethGenesisSpecBuiltin{Name: "ecrecover", + Linear: &alethGenesisSpecLinearPricing{Base: 3000}}) + spec.setPrecompile(2, &alethGenesisSpecBuiltin{Name: "sha256", + Linear: &alethGenesisSpecLinearPricing{Base: 60, Word: 12}}) + spec.setPrecompile(3, &alethGenesisSpecBuiltin{Name: "ripemd160", + Linear: &alethGenesisSpecLinearPricing{Base: 600, Word: 120}}) + spec.setPrecompile(4, &alethGenesisSpecBuiltin{Name: "identity", + Linear: &alethGenesisSpecLinearPricing{Base: 15, Word: 3}}) if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "modexp", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } - spec.Accounts[common.BytesToAddress([]byte{6})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_add", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 500}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_G1_mul", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), Linear: &cppEthereumGenesisSpecLinearPricing{Base: 40000}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Precompiled = &cppEthereumGenesisSpecBuiltin{ - Name: "alt_bn128_pairing_product", StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), - } + spec.setPrecompile(5, &alethGenesisSpecBuiltin{Name: "modexp", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) + spec.setPrecompile(6, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_add", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 500}}) + spec.setPrecompile(7, &alethGenesisSpecBuiltin{Name: "alt_bn128_G1_mul", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64()), + Linear: &alethGenesisSpecLinearPricing{Base: 40000}}) + spec.setPrecompile(8, &alethGenesisSpecBuiltin{Name: "alt_bn128_pairing_product", + StartingBlock: (hexutil.Uint64)(genesis.Config.ByzantiumBlock.Uint64())}) } return spec, nil } +func (spec *alethGenesisSpec) setPrecompile(address byte, data *alethGenesisSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) + } + spec.Accounts[common.UnprefixedAddress(common.BytesToAddress([]byte{address}))].Precompiled = data +} + +func (spec *alethGenesisSpec) setAccount(address common.Address, account core.GenesisAccount) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*alethGenesisSpecAccount) + } + + a, exist := spec.Accounts[common.UnprefixedAddress(address)] + if !exist { + a = &alethGenesisSpecAccount{} + spec.Accounts[common.UnprefixedAddress(address)] = a + } + a.Balance = (*math2.HexOrDecimal256)(account.Balance) + a.Nonce = account.Nonce + +} + +func (spec *alethGenesisSpec) setByzantium(num *big.Int) { + spec.Params.ByzantiumForkBlock = hexutil.Uint64(num.Uint64()) +} + +func (spec *alethGenesisSpec) setConstantinople(num *big.Int) { + spec.Params.ConstantinopleForkBlock = hexutil.Uint64(num.Uint64()) +} + // parityChainSpec is the chain specification format used by Parity. type parityChainSpec struct { - Name string `json:"name"` - Engine struct { + Name string `json:"name"` + Datadir string `json:"dataDir"` + Engine struct { Ethash struct { Params struct { - MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` - DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` - DurationLimit *hexutil.Big `json:"durationLimit"` - BlockReward *hexutil.Big `json:"blockReward"` - HomesteadTransition uint64 `json:"homesteadTransition"` - EIP150Transition uint64 `json:"eip150Transition"` - EIP160Transition uint64 `json:"eip160Transition"` - EIP161abcTransition uint64 `json:"eip161abcTransition"` - EIP161dTransition uint64 `json:"eip161dTransition"` - EIP649Reward *hexutil.Big `json:"eip649Reward"` - EIP100bTransition uint64 `json:"eip100bTransition"` - EIP649Transition uint64 `json:"eip649Transition"` + MinimumDifficulty *hexutil.Big `json:"minimumDifficulty"` + DifficultyBoundDivisor *hexutil.Big `json:"difficultyBoundDivisor"` + DurationLimit *hexutil.Big `json:"durationLimit"` + BlockReward map[string]string `json:"blockReward"` + DifficultyBombDelays map[string]string `json:"difficultyBombDelays"` + HomesteadTransition hexutil.Uint64 `json:"homesteadTransition"` + EIP100bTransition hexutil.Uint64 `json:"eip100bTransition"` } `json:"params"` } `json:"Ethash"` } `json:"engine"` Params struct { - MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` - MinGasLimit hexutil.Uint64 `json:"minGasLimit"` - GasLimitBoundDivisor hexutil.Uint64 `json:"gasLimitBoundDivisor"` - NetworkID hexutil.Uint64 `json:"networkID"` - MaxCodeSize uint64 `json:"maxCodeSize"` - EIP155Transition uint64 `json:"eip155Transition"` - EIP98Transition uint64 `json:"eip98Transition"` - EIP86Transition uint64 `json:"eip86Transition"` - EIP140Transition uint64 `json:"eip140Transition"` - EIP211Transition uint64 `json:"eip211Transition"` - EIP214Transition uint64 `json:"eip214Transition"` - EIP658Transition uint64 `json:"eip658Transition"` + AccountStartNonce hexutil.Uint64 `json:"accountStartNonce"` + MaximumExtraDataSize hexutil.Uint64 `json:"maximumExtraDataSize"` + MinGasLimit hexutil.Uint64 `json:"minGasLimit"` + GasLimitBoundDivisor math2.HexOrDecimal64 `json:"gasLimitBoundDivisor"` + NetworkID hexutil.Uint64 `json:"networkID"` + ChainID hexutil.Uint64 `json:"chainID"` + MaxCodeSize hexutil.Uint64 `json:"maxCodeSize"` + MaxCodeSizeTransition hexutil.Uint64 `json:"maxCodeSizeTransition"` + EIP98Transition hexutil.Uint64 `json:"eip98Transition"` + EIP150Transition hexutil.Uint64 `json:"eip150Transition"` + EIP160Transition hexutil.Uint64 `json:"eip160Transition"` + EIP161abcTransition hexutil.Uint64 `json:"eip161abcTransition"` + EIP161dTransition hexutil.Uint64 `json:"eip161dTransition"` + EIP155Transition hexutil.Uint64 `json:"eip155Transition"` + EIP140Transition hexutil.Uint64 `json:"eip140Transition"` + EIP211Transition hexutil.Uint64 `json:"eip211Transition"` + EIP214Transition hexutil.Uint64 `json:"eip214Transition"` + EIP658Transition hexutil.Uint64 `json:"eip658Transition"` + EIP145Transition hexutil.Uint64 `json:"eip145Transition"` + EIP1014Transition hexutil.Uint64 `json:"eip1014Transition"` + EIP1052Transition hexutil.Uint64 `json:"eip1052Transition"` + EIP1283Transition hexutil.Uint64 `json:"eip1283Transition"` } `json:"params"` Genesis struct { @@ -215,22 +259,22 @@ type parityChainSpec struct { GasLimit hexutil.Uint64 `json:"gasLimit"` } `json:"genesis"` - Nodes []string `json:"nodes"` - Accounts map[common.Address]*parityChainSpecAccount `json:"accounts"` + Nodes []string `json:"nodes"` + Accounts map[common.UnprefixedAddress]*parityChainSpecAccount `json:"accounts"` } // parityChainSpecAccount is the prefunded genesis account and/or precompiled // contract definition. type parityChainSpecAccount struct { - Balance *hexutil.Big `json:"balance"` - Nonce uint64 `json:"nonce,omitempty"` + Balance math2.HexOrDecimal256 `json:"balance"` + Nonce math2.HexOrDecimal64 `json:"nonce,omitempty"` Builtin *parityChainSpecBuiltin `json:"builtin,omitempty"` } // parityChainSpecBuiltin is the precompiled contract definition. type parityChainSpecBuiltin struct { Name string `json:"name,omitempty"` - ActivateAt uint64 `json:"activate_at,omitempty"` + ActivateAt math2.HexOrDecimal64 `json:"activate_at,omitempty"` Pricing *parityChainSpecPricing `json:"pricing,omitempty"` } @@ -265,34 +309,51 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin } // Reconstruct the chain spec in Parity's format spec := &parityChainSpec{ - Name: network, - Nodes: bootnodes, + Name: network, + Nodes: bootnodes, + Datadir: strings.ToLower(network), } + spec.Engine.Ethash.Params.BlockReward = make(map[string]string) + spec.Engine.Ethash.Params.DifficultyBombDelays = make(map[string]string) + // Frontier spec.Engine.Ethash.Params.MinimumDifficulty = (*hexutil.Big)(params.MinimumDifficulty) spec.Engine.Ethash.Params.DifficultyBoundDivisor = (*hexutil.Big)(params.DifficultyBoundDivisor) spec.Engine.Ethash.Params.DurationLimit = (*hexutil.Big)(params.DurationLimit) - spec.Engine.Ethash.Params.BlockReward = (*hexutil.Big)(ethash.FrontierBlockReward) - spec.Engine.Ethash.Params.HomesteadTransition = genesis.Config.HomesteadBlock.Uint64() - spec.Engine.Ethash.Params.EIP150Transition = genesis.Config.EIP150Block.Uint64() - spec.Engine.Ethash.Params.EIP160Transition = genesis.Config.EIP155Block.Uint64() - spec.Engine.Ethash.Params.EIP161abcTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP161dTransition = genesis.Config.EIP158Block.Uint64() - spec.Engine.Ethash.Params.EIP649Reward = (*hexutil.Big)(ethash.ByzantiumBlockReward) - spec.Engine.Ethash.Params.EIP100bTransition = genesis.Config.ByzantiumBlock.Uint64() - spec.Engine.Ethash.Params.EIP649Transition = genesis.Config.ByzantiumBlock.Uint64() + spec.Engine.Ethash.Params.BlockReward["0x0"] = hexutil.EncodeBig(ethash.FrontierBlockReward) + // Homestead + spec.Engine.Ethash.Params.HomesteadTransition = hexutil.Uint64(genesis.Config.HomesteadBlock.Uint64()) + + // Tangerine Whistle : 150 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-608.md + spec.Params.EIP150Transition = hexutil.Uint64(genesis.Config.EIP150Block.Uint64()) + + // Spurious Dragon: 155, 160, 161, 170 + // https://github.com/ethereum/EIPs/blob/master/EIPS/eip-607.md + spec.Params.EIP155Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP160Transition = hexutil.Uint64(genesis.Config.EIP155Block.Uint64()) + spec.Params.EIP161abcTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + spec.Params.EIP161dTransition = hexutil.Uint64(genesis.Config.EIP158Block.Uint64()) + + // Byzantium + if num := genesis.Config.ByzantiumBlock; num != nil { + spec.setByzantium(num) + } + // Constantinople + if num := genesis.Config.ConstantinopleBlock; num != nil { + spec.setConstantinople(num) + } spec.Params.MaximumExtraDataSize = (hexutil.Uint64)(params.MaximumExtraDataSize) spec.Params.MinGasLimit = (hexutil.Uint64)(params.MinGasLimit) - spec.Params.GasLimitBoundDivisor = (hexutil.Uint64)(params.GasLimitBoundDivisor) + spec.Params.GasLimitBoundDivisor = (math2.HexOrDecimal64)(params.GasLimitBoundDivisor) spec.Params.NetworkID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) + spec.Params.ChainID = (hexutil.Uint64)(genesis.Config.ChainID.Uint64()) spec.Params.MaxCodeSize = params.MaxCodeSize - spec.Params.EIP155Transition = genesis.Config.EIP155Block.Uint64() - spec.Params.EIP98Transition = math.MaxUint64 - spec.Params.EIP86Transition = math.MaxUint64 - spec.Params.EIP140Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP211Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP214Transition = genesis.Config.ByzantiumBlock.Uint64() - spec.Params.EIP658Transition = genesis.Config.ByzantiumBlock.Uint64() + // geth has it set from zero + spec.Params.MaxCodeSizeTransition = 0 + + // Disable this one + spec.Params.EIP98Transition = math.MaxInt64 spec.Genesis.Seal.Ethereum.Nonce = (hexutil.Bytes)(make([]byte, 8)) binary.LittleEndian.PutUint64(spec.Genesis.Seal.Ethereum.Nonce[:], genesis.Nonce) @@ -305,42 +366,77 @@ func newParityChainSpec(network string, genesis *core.Genesis, bootnodes []strin spec.Genesis.ExtraData = (hexutil.Bytes)(genesis.ExtraData) spec.Genesis.GasLimit = (hexutil.Uint64)(genesis.GasLimit) - spec.Accounts = make(map[common.Address]*parityChainSpecAccount) + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) for address, account := range genesis.Alloc { - spec.Accounts[address] = &parityChainSpecAccount{ - Balance: (*hexutil.Big)(account.Balance), - Nonce: account.Nonce, + bal := math2.HexOrDecimal256(*account.Balance) + + spec.Accounts[common.UnprefixedAddress(address)] = &parityChainSpecAccount{ + Balance: bal, + Nonce: math2.HexOrDecimal64(account.Nonce), } } - spec.Accounts[common.BytesToAddress([]byte{1})].Builtin = &parityChainSpecBuiltin{ - Name: "ecrecover", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}, - } - spec.Accounts[common.BytesToAddress([]byte{2})].Builtin = &parityChainSpecBuiltin{ + spec.setPrecompile(1, &parityChainSpecBuiltin{Name: "ecrecover", + Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 3000}}}) + + spec.setPrecompile(2, &parityChainSpecBuiltin{ Name: "sha256", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 60, Word: 12}}, - } - spec.Accounts[common.BytesToAddress([]byte{3})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(3, &parityChainSpecBuiltin{ Name: "ripemd160", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 600, Word: 120}}, - } - spec.Accounts[common.BytesToAddress([]byte{4})].Builtin = &parityChainSpecBuiltin{ + }) + spec.setPrecompile(4, &parityChainSpecBuiltin{ Name: "identity", Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 15, Word: 3}}, - } + }) if genesis.Config.ByzantiumBlock != nil { - spec.Accounts[common.BytesToAddress([]byte{5})].Builtin = &parityChainSpecBuiltin{ - Name: "modexp", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, - } - spec.Accounts[common.BytesToAddress([]byte{6})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_add", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, - } - spec.Accounts[common.BytesToAddress([]byte{7})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_mul", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, - } - spec.Accounts[common.BytesToAddress([]byte{8})].Builtin = &parityChainSpecBuiltin{ - Name: "alt_bn128_pairing", ActivateAt: genesis.Config.ByzantiumBlock.Uint64(), Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, - } + blnum := math2.HexOrDecimal64(genesis.Config.ByzantiumBlock.Uint64()) + spec.setPrecompile(5, &parityChainSpecBuiltin{ + Name: "modexp", ActivateAt: blnum, Pricing: &parityChainSpecPricing{ModExp: &parityChainSpecModExpPricing{Divisor: 20}}, + }) + spec.setPrecompile(6, &parityChainSpecBuiltin{ + Name: "alt_bn128_add", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 500}}, + }) + spec.setPrecompile(7, &parityChainSpecBuiltin{ + Name: "alt_bn128_mul", ActivateAt: blnum, Pricing: &parityChainSpecPricing{Linear: &parityChainSpecLinearPricing{Base: 40000}}, + }) + spec.setPrecompile(8, &parityChainSpecBuiltin{ + Name: "alt_bn128_pairing", ActivateAt: blnum, Pricing: &parityChainSpecPricing{AltBnPairing: &parityChainSpecAltBnPairingPricing{Base: 100000, Pair: 80000}}, + }) } return spec, nil } +func (spec *parityChainSpec) setPrecompile(address byte, data *parityChainSpecBuiltin) { + if spec.Accounts == nil { + spec.Accounts = make(map[common.UnprefixedAddress]*parityChainSpecAccount) + } + a := common.UnprefixedAddress(common.BytesToAddress([]byte{address})) + if _, exist := spec.Accounts[a]; !exist { + spec.Accounts[a] = &parityChainSpecAccount{} + } + spec.Accounts[a].Builtin = data +} + +func (spec *parityChainSpec) setByzantium(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ByzantiumBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(3000000) + n := hexutil.Uint64(num.Uint64()) + spec.Engine.Ethash.Params.EIP100bTransition = n + spec.Params.EIP140Transition = n + spec.Params.EIP211Transition = n + spec.Params.EIP214Transition = n + spec.Params.EIP658Transition = n +} + +func (spec *parityChainSpec) setConstantinople(num *big.Int) { + spec.Engine.Ethash.Params.BlockReward[hexutil.EncodeBig(num)] = hexutil.EncodeBig(ethash.ConstantinopleBlockReward) + spec.Engine.Ethash.Params.DifficultyBombDelays[hexutil.EncodeBig(num)] = hexutil.EncodeUint64(2000000) + n := hexutil.Uint64(num.Uint64()) + spec.Params.EIP145Transition = n + spec.Params.EIP1014Transition = n + spec.Params.EIP1052Transition = n + spec.Params.EIP1283Transition = n +} + // pyEthereumGenesisSpec represents the genesis specification format used by the // Python Ethereum implementation. type pyEthereumGenesisSpec struct { diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index d22bd81103..cb3ed6e71c 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -640,7 +640,7 @@ func deployDashboard(client *sshClient, network string, conf *config, config *da files[filepath.Join(workdir, network+".json")] = genesis if conf.Genesis.Config.Ethash != nil { - cppSpec, err := newCppEthereumGenesisSpec(network, conf.Genesis) + cppSpec, err := newAlethGenesisSpec(network, conf.Genesis) if err != nil { return nil, err } diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index df3fff51b0..3af1751eae 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,11 +18,15 @@ package main import ( + "encoding/json" + "io/ioutil" "math/rand" "os" "strings" "time" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -43,6 +47,14 @@ func main() { Usage: "log level to emit to the screen", }, } + app.Commands = []cli.Command{ + cli.Command{ + Action: utils.MigrateFlags(convert), + Name: "convert", + Usage: "Convert from geth genesis into chainspecs for other nodes.", + ArgsUsage: "", + }, + } app.Action = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) @@ -58,3 +70,23 @@ func main() { } app.Run(os.Args) } + +func convert(ctx *cli.Context) error { + // Ensure we have a source genesis + log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) + if len(ctx.Args()) != 1 { + utils.Fatalf("No geth genesis provided") + } + blob, err := ioutil.ReadFile(ctx.Args().First()) + if err != nil { + utils.Fatalf("Could not read file: %v", err) + } + + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + utils.Fatalf("Failed parsing genesis: %v", err) + } + basename := strings.TrimRight(ctx.Args().First(), ".json") + convertGenesis(&genesis, basename, basename, []string{}) + return nil +} diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/puppeth_test.go new file mode 100644 index 0000000000..ae0752d549 --- /dev/null +++ b/cmd/puppeth/puppeth_test.go @@ -0,0 +1,91 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "reflect" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/core" +) + +func TestConverter_AlethStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newAlethGenesisSpec("stureby", &genesis) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_aleth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &alethGenesisSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} + +func TestConverter_ParityStureby(t *testing.T) { + blob, err := ioutil.ReadFile("testdata/stureby_geth.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + var genesis core.Genesis + if err := json.Unmarshal(blob, &genesis); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + spec, err := newParityChainSpec("Stureby", &genesis, []string{}) + if err != nil { + t.Fatalf("failed creating chainspec: %v", err) + } + + expBlob, err := ioutil.ReadFile("testdata/stureby_parity.json") + if err != nil { + t.Fatalf("could not read file: %v", err) + } + expspec := &parityChainSpec{} + if err := json.Unmarshal(expBlob, expspec); err != nil { + t.Fatalf("failed parsing genesis: %v", err) + } + expspec.Nodes = []string{} + + if !reflect.DeepEqual(expspec, spec) { + t.Errorf("chainspec mismatch") + c := spew.ConfigState{ + DisablePointerAddresses: true, + SortKeys: true, + } + exp := strings.Split(c.Sdump(expspec), "\n") + got := strings.Split(c.Sdump(spec), "\n") + for i := 0; i < len(exp) && i < len(got); i++ { + if exp[i] != got[i] { + fmt.Printf("got: %v\nexp: %v\n", exp[i], got[i]) + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_aleth.json b/cmd/puppeth/testdata/stureby_aleth.json new file mode 100644 index 0000000000..1ef1d8ae18 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_aleth.json @@ -0,0 +1,112 @@ +{ + "sealEngine":"Ethash", + "params":{ + "accountStartNonce":"0x00", + "maximumExtraDataSize":"0x20", + "homesteadForkBlock":"0x2710", + "daoHardforkBlock":"0x00", + "EIP150ForkBlock":"0x3a98", + "EIP158ForkBlock":"0x59d8", + "byzantiumForkBlock":"0x7530", + "constantinopleForkBlock":"0x9c40", + "minGasLimit":"0x1388", + "maxGasLimit":"0x7fffffffffffffff", + "tieBreakingGas":false, + "gasLimitBoundDivisor":"0x0400", + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x0800", + "durationLimit":"0x0d", + "blockReward":"0x4563918244F40000", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "allowFutureBlocks":false + }, + "genesis":{ + "nonce":"0x0000000000000000", + "difficulty":"0x20000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "precompiled":{ + "name":"ecrecover", + "linear":{ + "base":3000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "precompiled":{ + "name":"sha256", + "linear":{ + "base":60, + "word":12 + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "precompiled":{ + "name":"ripemd160", + "linear":{ + "base":600, + "word":120 + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "precompiled":{ + "name":"identity", + "linear":{ + "base":15, + "word":3 + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "precompiled":{ + "name":"modexp", + "startingBlock":"0x7530" + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_add", + "startingBlock":"0x7530", + "linear":{ + "base":500, + "word":0 + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_G1_mul", + "startingBlock":"0x7530", + "linear":{ + "base":40000, + "word":0 + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "precompiled":{ + "name":"alt_bn128_pairing_product", + "startingBlock":"0x7530" + } + } + } +} diff --git a/cmd/puppeth/testdata/stureby_geth.json b/cmd/puppeth/testdata/stureby_geth.json new file mode 100644 index 0000000000..c8c3b3c956 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_geth.json @@ -0,0 +1,47 @@ +{ + "config": { + "ethash":{}, + "chainId": 314158, + "homesteadBlock": 10000, + "eip150Block": 15000, + "eip150Hash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "eip155Block": 23000, + "eip158Block": 23000, + "byzantiumBlock": 30000, + "constantinopleBlock": 40000 + }, + "nonce": "0x0", + "timestamp": "0x59a4e76d", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData": "0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit": "0x47b760", + "difficulty": "0x20000", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "alloc": { + "0000000000000000000000000000000000000001": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000002": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000003": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000004": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000005": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000006": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000007": { + "balance": "0x01" + }, + "0000000000000000000000000000000000000008": { + "balance": "0x01" + } + } +} diff --git a/cmd/puppeth/testdata/stureby_parity.json b/cmd/puppeth/testdata/stureby_parity.json new file mode 100644 index 0000000000..f3fa8386a0 --- /dev/null +++ b/cmd/puppeth/testdata/stureby_parity.json @@ -0,0 +1,181 @@ +{ + "name":"Stureby", + "dataDir":"stureby", + "engine":{ + "Ethash":{ + "params":{ + "minimumDifficulty":"0x20000", + "difficultyBoundDivisor":"0x800", + "durationLimit":"0xd", + "blockReward":{ + "0x0":"0x4563918244f40000", + "0x7530":"0x29a2241af62c0000", + "0x9c40":"0x1bc16d674ec80000" + }, + "homesteadTransition":"0x2710", + "eip100bTransition":"0x7530", + "difficultyBombDelays":{ + "0x7530":"0x2dc6c0", + "0x9c40":"0x1e8480" + } + } + } + }, + "params":{ + "accountStartNonce":"0x0", + "maximumExtraDataSize":"0x20", + "gasLimitBoundDivisor":"0x400", + "minGasLimit":"0x1388", + "networkID":"0x4cb2e", + "chainID":"0x4cb2e", + "maxCodeSize":"0x6000", + "maxCodeSizeTransition":"0x0", + "eip98Transition": "0x7fffffffffffffff", + "eip150Transition":"0x3a98", + "eip160Transition":"0x59d8", + "eip161abcTransition":"0x59d8", + "eip161dTransition":"0x59d8", + "eip155Transition":"0x59d8", + "eip140Transition":"0x7530", + "eip211Transition":"0x7530", + "eip214Transition":"0x7530", + "eip658Transition":"0x7530", + "eip145Transition":"0x9c40", + "eip1014Transition":"0x9c40", + "eip1052Transition":"0x9c40", + "eip1283Transition":"0x9c40" + }, + "genesis":{ + "seal":{ + "ethereum":{ + "nonce":"0x0000000000000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000" + } + }, + "difficulty":"0x20000", + "author":"0x0000000000000000000000000000000000000000", + "timestamp":"0x59a4e76d", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "extraData":"0x0000000000000000000000000000000000000000000000000000000b4dc0ffee", + "gasLimit":"0x47b760" + }, + "nodes":[ + "enode://dfa7aca3f5b635fbfe7d0b20575f25e40d9e27b4bfbb3cf74364a42023ad9f25c1a4383bcc8cced86ee511a7d03415345a4df05be37f1dff040e4c780699f1c0@168.61.153.255:31303", + "enode://ef441b20dd70aeabf0eac35c3b8a2854e5ce04db0e30be9152ea9fd129359dcbb3f803993303ff5781c755dfd7223f3fe43505f583cccb740949407677412ba9@40.74.91.252:31303", + "enode://953b5ea1c8987cf46008232a0160324fd00d41320ecf00e23af86ec8f5396b19eb57ddab37c78141be56f62e9077de4f4dfa0747fa768ed8c8531bbfb1046237@40.70.214.166:31303", + "enode://276e613dd4b277a66591e565711e6c8bb107f0905248a9f8f8228c1a87992e156e5114bb9937c02824a9d9d25f76340442cf86e2028bf5293cae19904fb2b98e@35.178.251.52:30303", + "enode://064c820d41e52ed7d426ac64b60506c2998235bedc7e67cb497c6faf7bb4fc54fe56fc82d0add3180b747c0c4f40a1108a6f84d7d0629ed606d504528e61cc57@3.8.5.3:30303", + "enode://90069fdabcc5e684fa5d59430bebbb12755d9362dfe5006a1485b13d71a78a3812d36e74dd7d88e50b51add01e097ea80f16263aeaa4f0230db6c79e2a97e7ca@217.29.191.142:30303", + "enode://0aac74b7fd28726275e466acb5e03bc88a95927e9951eb66b5efb239b2f798ada0690853b2f2823fe4efa408f0f3d4dd258430bc952a5ff70677b8625b3e3b14@40.115.33.57:40404", + "enode://0b96415a10f835106d83e090a0528eed5e7887e5c802a6d084e9f1993a9d0fc713781e6e4101f6365e9b91259712f291acc0a9e6e667e22023050d602c36fbe2@40.115.33.57:40414" + ], + "accounts":{ + "0000000000000000000000000000000000000001":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ecrecover", + "pricing":{ + "linear":{ + "base":3000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000002":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"sha256", + "pricing":{ + "linear":{ + "base":60, + "word":12 + } + } + } + }, + "0000000000000000000000000000000000000003":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"ripemd160", + "pricing":{ + "linear":{ + "base":600, + "word":120 + } + } + } + }, + "0000000000000000000000000000000000000004":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"identity", + "pricing":{ + "linear":{ + "base":15, + "word":3 + } + } + } + }, + "0000000000000000000000000000000000000005":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"modexp", + "activate_at":"0x7530", + "pricing":{ + "modexp":{ + "divisor":20 + } + } + } + }, + "0000000000000000000000000000000000000006":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_add", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":500, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000007":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_mul", + "activate_at":"0x7530", + "pricing":{ + "linear":{ + "base":40000, + "word":0 + } + } + } + }, + "0000000000000000000000000000000000000008":{ + "balance":"1", + "nonce":"0", + "builtin":{ + "name":"alt_bn128_pairing", + "activate_at":"0x7530", + "pricing":{ + "alt_bn128_pairing":{ + "base":100000, + "pair":80000 + } + } + } + } + } +} diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index b88a61de7d..42fc125e58 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -104,6 +104,28 @@ func (w *wizard) readString() string { } } +// readYesNo reads a yes or no from stdin, returning boolean true for yes +func (w *wizard) readYesNo(def bool) bool { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + text = strings.ToLower(strings.TrimSpace(text)) + if text == "y" || text == "yes" { + return true + } + if text == "n" || text == "no" { + return false + } + if len(text) == 0 { + return def + } + fmt.Println("Valid answers: y, yes, n, no or leave empty for default") + } +} + // readDefaultString reads a single line from stdin, trimming if from spaces. If // an empty line is entered, the default value is returned. func (w *wizard) readDefaultString(def string) string { diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 6c4cd571fb..681a387dff 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -114,9 +114,13 @@ func (w *wizard) makeGenesis() { } break } - // Add a batch of precompile balances to avoid them getting deleted - for i := int64(0); i < 256; i++ { - genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + fmt.Println() + fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") + if w.readYesNo(true) { + // Add a batch of precompile balances to avoid them getting deleted + for i := int64(0); i < 256; i++ { + genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} + } } // Query the user for some custom extras fmt.Println() @@ -136,47 +140,57 @@ func (w *wizard) manageGenesis() { // Figure out whether to modify or export the genesis fmt.Println() fmt.Println(" 1. Modify existing fork rules") - fmt.Println(" 2. Export genesis configuration") + fmt.Println(" 2. Export genesis configurations") fmt.Println(" 3. Remove genesis configuration") choice := w.read() - switch { - case choice == "1": + switch choice { + case "1": // Fork rule updating requested, iterate over each fork fmt.Println() fmt.Printf("Which block should Homestead come into effect? (default = %v)\n", w.conf.Genesis.Config.HomesteadBlock) w.conf.Genesis.Config.HomesteadBlock = w.readDefaultBigInt(w.conf.Genesis.Config.HomesteadBlock) fmt.Println() - fmt.Printf("Which block should EIP150 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) + fmt.Printf("Which block should EIP150 (Tangerine Whistle) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP150Block) w.conf.Genesis.Config.EIP150Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP150Block) fmt.Println() - fmt.Printf("Which block should EIP155 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) + fmt.Printf("Which block should EIP155 (Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP155Block) w.conf.Genesis.Config.EIP155Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP155Block) fmt.Println() - fmt.Printf("Which block should EIP158 come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) + fmt.Printf("Which block should EIP158/161 (also Spurious Dragon) come into effect? (default = %v)\n", w.conf.Genesis.Config.EIP158Block) w.conf.Genesis.Config.EIP158Block = w.readDefaultBigInt(w.conf.Genesis.Config.EIP158Block) fmt.Println() fmt.Printf("Which block should Byzantium come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) + fmt.Println() + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) + out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") fmt.Printf("Chain configuration updated:\n\n%s\n", out) - case choice == "2": + case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which file to save the genesis into? (default = %s.json)\n", w.network) + fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - if err := ioutil.WriteFile(w.readDefaultString(fmt.Sprintf("%s.json", w.network)), out, 0644); err != nil { + basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) + + gethJson := fmt.Sprintf("%s.json", basename) + if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) } - log.Info("Exported existing genesis block") + log.Info("Saved geth genesis as %v", gethJson) + if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { + log.Error("Conversion failed", "err", err) + } - case choice == "3": + case "3": // Make sure we don't have any services running if len(w.conf.servers()) > 0 { log.Error("Genesis reset requires all services and servers torn down") @@ -186,8 +200,31 @@ func (w *wizard) manageGenesis() { w.conf.Genesis = nil w.conf.flush() - default: log.Error("That's not something I can do") } } + +func saveGenesis(basename, client string, spec interface{}) { + filename := fmt.Sprintf("%s-%s.json", basename, client) + out, _ := json.Marshal(spec) + if err := ioutil.WriteFile(filename, out, 0644); err != nil { + log.Error("failed to save genesis file", "client", client, "err", err) + } + log.Info("saved chainspec", "client", client, "filename", filename) +} + +func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { + if spec, err := newAlethGenesisSpec(network, genesis); err == nil { + saveGenesis(basename, "aleth", spec) + } else { + log.Error("failed to create chain spec", "client", "aleth", "err", err) + } + if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { + saveGenesis(basename, "parity", spec) + } else { + log.Error("failed to create chain spec", "client", "parity", "err", err) + } + saveGenesis(basename, "harmony", genesis) + return nil +} From 7a5c1b28ddd4ce1e95f6ed01672b9d44dba047f2 Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Mon, 3 Dec 2018 21:15:22 +0800 Subject: [PATCH 26/68] whisperv6: remove duplicated code (#18015) --- whisper/whisperv6/api_test.go | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/whisper/whisperv6/api_test.go b/whisper/whisperv6/api_test.go index cdbc7fab5e..6d7157f57f 100644 --- a/whisper/whisperv6/api_test.go +++ b/whisper/whisperv6/api_test.go @@ -18,27 +18,12 @@ package whisperv6 import ( "bytes" - "crypto/ecdsa" "testing" "time" - - mapset "github.com/deckarep/golang-set" - "github.com/ethereum/go-ethereum/common" ) func TestMultipleTopicCopyInNewMessageFilter(t *testing.T) { - w := &Whisper{ - privateKeys: make(map[string]*ecdsa.PrivateKey), - symKeys: make(map[string][]byte), - envelopes: make(map[common.Hash]*Envelope), - expirations: make(map[uint32]mapset.Set), - peers: make(map[*Peer]struct{}), - messageQueue: make(chan *Envelope, messageQueueLimit), - p2pMsgQueue: make(chan *Envelope, messageQueueLimit), - quit: make(chan struct{}), - syncAllowance: DefaultSyncAllowance, - } - w.filters = NewFilters(w) + w := New(nil) keyID, err := w.GenerateSymKey() if err != nil { From d4415f5e40fbd155adb3e85e4eae5c7db7832694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 3 Dec 2018 16:50:59 +0200 Subject: [PATCH 27/68] cmd/puppeth: chain import/export via wizard, minor polishes --- .../{puppeth_test.go => genesis_test.go} | 22 ++- cmd/puppeth/puppeth.go | 43 ++---- cmd/puppeth/wizard.go | 64 ++++++--- cmd/puppeth/wizard_dashboard.go | 4 +- cmd/puppeth/wizard_ethstats.go | 6 +- cmd/puppeth/wizard_explorer.go | 2 +- cmd/puppeth/wizard_faucet.go | 8 +- cmd/puppeth/wizard_genesis.go | 129 +++++++++++++----- cmd/puppeth/wizard_intro.go | 16 ++- cmd/puppeth/wizard_nginx.go | 4 +- cmd/puppeth/wizard_node.go | 4 +- cmd/puppeth/wizard_wallet.go | 2 +- 12 files changed, 194 insertions(+), 110 deletions(-) rename cmd/puppeth/{puppeth_test.go => genesis_test.go} (70%) diff --git a/cmd/puppeth/puppeth_test.go b/cmd/puppeth/genesis_test.go similarity index 70% rename from cmd/puppeth/puppeth_test.go rename to cmd/puppeth/genesis_test.go index ae0752d549..83e7383605 100644 --- a/cmd/puppeth/puppeth_test.go +++ b/cmd/puppeth/genesis_test.go @@ -1,3 +1,19 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + package main import ( @@ -12,7 +28,8 @@ import ( "github.com/ethereum/go-ethereum/core" ) -func TestConverter_AlethStureby(t *testing.T) { +// Tests the go-ethereum to Aleth chainspec conversion for the Stureby testnet. +func TestAlethSturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) @@ -50,7 +67,8 @@ func TestConverter_AlethStureby(t *testing.T) { } } -func TestConverter_ParityStureby(t *testing.T) { +// Tests the go-ethereum to Parity chainspec conversion for the Stureby testnet. +func TestParitySturebyConverter(t *testing.T) { blob, err := ioutil.ReadFile("testdata/stureby_geth.json") if err != nil { t.Fatalf("could not read file: %v", err) diff --git a/cmd/puppeth/puppeth.go b/cmd/puppeth/puppeth.go index 3af1751eae..c3de5f9360 100644 --- a/cmd/puppeth/puppeth.go +++ b/cmd/puppeth/puppeth.go @@ -18,15 +18,11 @@ package main import ( - "encoding/json" - "io/ioutil" "math/rand" "os" "strings" "time" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/log" "gopkg.in/urfave/cli.v1" ) @@ -47,46 +43,23 @@ func main() { Usage: "log level to emit to the screen", }, } - app.Commands = []cli.Command{ - cli.Command{ - Action: utils.MigrateFlags(convert), - Name: "convert", - Usage: "Convert from geth genesis into chainspecs for other nodes.", - ArgsUsage: "", - }, - } - app.Action = func(c *cli.Context) error { + app.Before = func(c *cli.Context) error { // Set up the logger to print everything and the random generator log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int("loglevel")), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) rand.Seed(time.Now().UnixNano()) - network := c.String("network") - if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { - log.Crit("No spaces, hyphens or capital letters allowed in network name") - } - // Start the wizard and relinquish control - makeWizard(c.String("network")).run() return nil } + app.Action = runWizard app.Run(os.Args) } -func convert(ctx *cli.Context) error { - // Ensure we have a source genesis - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - if len(ctx.Args()) != 1 { - utils.Fatalf("No geth genesis provided") +// runWizard start the wizard and relinquish control to it. +func runWizard(c *cli.Context) error { + network := c.String("network") + if strings.Contains(network, " ") || strings.Contains(network, "-") || strings.ToLower(network) != network { + log.Crit("No spaces, hyphens or capital letters allowed in network name") } - blob, err := ioutil.ReadFile(ctx.Args().First()) - if err != nil { - utils.Fatalf("Could not read file: %v", err) - } - - var genesis core.Genesis - if err := json.Unmarshal(blob, &genesis); err != nil { - utils.Fatalf("Failed parsing genesis: %v", err) - } - basename := strings.TrimRight(ctx.Args().First(), ".json") - convertGenesis(&genesis, basename, basename, []string{}) + makeWizard(c.String("network")).run() return nil } diff --git a/cmd/puppeth/wizard.go b/cmd/puppeth/wizard.go index 42fc125e58..83536506c4 100644 --- a/cmd/puppeth/wizard.go +++ b/cmd/puppeth/wizard.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math/big" "net" + "net/url" "os" "path/filepath" "sort" @@ -104,28 +105,6 @@ func (w *wizard) readString() string { } } -// readYesNo reads a yes or no from stdin, returning boolean true for yes -func (w *wizard) readYesNo(def bool) bool { - for { - fmt.Printf("> ") - text, err := w.in.ReadString('\n') - if err != nil { - log.Crit("Failed to read user input", "err", err) - } - text = strings.ToLower(strings.TrimSpace(text)) - if text == "y" || text == "yes" { - return true - } - if text == "n" || text == "no" { - return false - } - if len(text) == 0 { - return def - } - fmt.Println("Valid answers: y, yes, n, no or leave empty for default") - } -} - // readDefaultString reads a single line from stdin, trimming if from spaces. If // an empty line is entered, the default value is returned. func (w *wizard) readDefaultString(def string) string { @@ -140,6 +119,47 @@ func (w *wizard) readDefaultString(def string) string { return def } +// readDefaultYesNo reads a single line from stdin, trimming if from spaces and +// interpreting it as a 'yes' or a 'no'. If an empty line is entered, the default +// value is returned. +func (w *wizard) readDefaultYesNo(def bool) bool { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + if text = strings.ToLower(strings.TrimSpace(text)); text == "" { + return def + } + if text == "y" || text == "yes" { + return true + } + if text == "n" || text == "no" { + return false + } + log.Error("Invalid input, expected 'y', 'yes', 'n', 'no' or empty") + } +} + +// readURL reads a single line from stdin, trimming if from spaces and trying to +// interpret it as a URL (http, https or file). +func (w *wizard) readURL() *url.URL { + for { + fmt.Printf("> ") + text, err := w.in.ReadString('\n') + if err != nil { + log.Crit("Failed to read user input", "err", err) + } + uri, err := url.Parse(strings.TrimSpace(text)) + if err != nil { + log.Error("Invalid input, expected URL", "err", err) + continue + } + return uri + } +} + // readInt reads a single line from stdin, trimming if from spaces, enforcing it // to parse into an integer. func (w *wizard) readInt() int { diff --git a/cmd/puppeth/wizard_dashboard.go b/cmd/puppeth/wizard_dashboard.go index 1a01631ff4..8a8370845b 100644 --- a/cmd/puppeth/wizard_dashboard.go +++ b/cmd/puppeth/wizard_dashboard.go @@ -137,14 +137,14 @@ func (w *wizard) deployDashboard() { if w.conf.ethstats != "" { fmt.Println() fmt.Println("Include ethstats secret on dashboard (y/n)? (default = yes)") - infos.trusted = w.readDefaultString("y") == "y" + infos.trusted = w.readDefaultYesNo(true) } // Try to deploy the dashboard container on the host nocache := false if existed { fmt.Println() fmt.Printf("Should the dashboard be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployDashboard(client, w.network, &w.conf, infos, nocache); err != nil { log.Error("Failed to deploy dashboard container", "err", err) diff --git a/cmd/puppeth/wizard_ethstats.go b/cmd/puppeth/wizard_ethstats.go index fb2529c267..58ff3efbe9 100644 --- a/cmd/puppeth/wizard_ethstats.go +++ b/cmd/puppeth/wizard_ethstats.go @@ -67,11 +67,11 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Keep existing IP %v blacklist (y/n)? (default = yes)\n", infos.banned) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { // The user might want to clear the entire list, although generally probably not fmt.Println() fmt.Printf("Clear out blacklist and start over (y/n)? (default = no)\n") - if w.readDefaultString("n") != "n" { + if w.readDefaultYesNo(false) { infos.banned = nil } // Offer the user to explicitly add/remove certain IP addresses @@ -106,7 +106,7 @@ func (w *wizard) deployEthstats() { if existed { fmt.Println() fmt.Printf("Should the ethstats be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } trusted := make([]string, 0, len(w.servers)) for _, client := range w.servers { diff --git a/cmd/puppeth/wizard_explorer.go b/cmd/puppeth/wizard_explorer.go index 413511c1c3..a128fb9fb5 100644 --- a/cmd/puppeth/wizard_explorer.go +++ b/cmd/puppeth/wizard_explorer.go @@ -100,7 +100,7 @@ func (w *wizard) deployExplorer() { if existed { fmt.Println() fmt.Printf("Should the explorer be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployExplorer(client, w.network, chain, infos, nocache); err != nil { log.Error("Failed to deploy explorer container", "err", err) diff --git a/cmd/puppeth/wizard_faucet.go b/cmd/puppeth/wizard_faucet.go index 6f08408947..9068c1d30b 100644 --- a/cmd/puppeth/wizard_faucet.go +++ b/cmd/puppeth/wizard_faucet.go @@ -81,7 +81,7 @@ func (w *wizard) deployFaucet() { if infos.captchaToken != "" { fmt.Println() fmt.Println("Reuse previous reCaptcha API authorization (y/n)? (default = yes)") - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.captchaToken, infos.captchaSecret = "", "" } } @@ -89,7 +89,7 @@ func (w *wizard) deployFaucet() { // No previous authorization (or old one discarded) fmt.Println() fmt.Println("Enable reCaptcha protection against robots (y/n)? (default = no)") - if w.readDefaultString("n") == "n" { + if !w.readDefaultYesNo(false) { log.Warn("Users will be able to requests funds via automated scripts") } else { // Captcha protection explicitly requested, read the site and secret keys @@ -132,7 +132,7 @@ func (w *wizard) deployFaucet() { } else { fmt.Println() fmt.Printf("Reuse previous (%s) funding account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.node.keyJSON, infos.node.keyPass = "", "" } } @@ -166,7 +166,7 @@ func (w *wizard) deployFaucet() { if existed { fmt.Println() fmt.Printf("Should the faucet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployFaucet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy faucet container", "err", err) diff --git a/cmd/puppeth/wizard_genesis.go b/cmd/puppeth/wizard_genesis.go index 681a387dff..95da5bd4f3 100644 --- a/cmd/puppeth/wizard_genesis.go +++ b/cmd/puppeth/wizard_genesis.go @@ -20,9 +20,13 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "math/big" "math/rand" + "net/http" + "os" + "path/filepath" "time" "github.com/ethereum/go-ethereum/common" @@ -40,11 +44,12 @@ func (w *wizard) makeGenesis() { Difficulty: big.NewInt(524288), Alloc: make(core.GenesisAlloc), Config: ¶ms.ChainConfig{ - HomesteadBlock: big.NewInt(1), - EIP150Block: big.NewInt(2), - EIP155Block: big.NewInt(3), - EIP158Block: big.NewInt(3), - ByzantiumBlock: big.NewInt(4), + HomesteadBlock: big.NewInt(1), + EIP150Block: big.NewInt(2), + EIP155Block: big.NewInt(3), + EIP158Block: big.NewInt(3), + ByzantiumBlock: big.NewInt(4), + ConstantinopleBlock: big.NewInt(5), }, } // Figure out which consensus engine to choose @@ -116,7 +121,7 @@ func (w *wizard) makeGenesis() { } fmt.Println() fmt.Println("Should the precompile-addresses (0x1 .. 0xff) be pre-funded with 1 wei? (advisable yes)") - if w.readYesNo(true) { + if w.readDefaultYesNo(true) { // Add a batch of precompile balances to avoid them getting deleted for i := int64(0); i < 256; i++ { genesis.Alloc[common.BigToAddress(big.NewInt(i))] = core.GenesisAccount{Balance: big.NewInt(1)} @@ -134,6 +139,53 @@ func (w *wizard) makeGenesis() { w.conf.flush() } +// importGenesis imports a Geth genesis spec into puppeth. +func (w *wizard) importGenesis() { + // Request the genesis JSON spec URL from the user + fmt.Println() + fmt.Println("Where's the genesis file? (local file or http/https url)") + url := w.readURL() + + // Convert the various allowed URLs to a reader stream + var reader io.Reader + + switch url.Scheme { + case "http", "https": + // Remote web URL, retrieve it via an HTTP client + res, err := http.Get(url.String()) + if err != nil { + log.Error("Failed to retrieve remote genesis", "err", err) + return + } + defer res.Body.Close() + reader = res.Body + + case "": + // Schemaless URL, interpret as a local file + file, err := os.Open(url.String()) + if err != nil { + log.Error("Failed to open local genesis", "err", err) + return + } + defer file.Close() + reader = file + + default: + log.Error("Unsupported genesis URL scheme", "scheme", url.Scheme) + return + } + // Parse the genesis file and inject it successful + var genesis core.Genesis + if err := json.NewDecoder(reader).Decode(&genesis); err != nil { + log.Error("Invalid genesis spec: %v", err) + return + } + log.Info("Imported genesis block") + + w.conf.Genesis = &genesis + w.conf.flush() +} + // manageGenesis permits the modification of chain configuration parameters in // a genesis config and the export of the entire genesis spec. func (w *wizard) manageGenesis() { @@ -168,7 +220,7 @@ func (w *wizard) manageGenesis() { w.conf.Genesis.Config.ByzantiumBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ByzantiumBlock) fmt.Println() - fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ByzantiumBlock) + fmt.Printf("Which block should Constantinople come into effect? (default = %v)\n", w.conf.Genesis.Config.ConstantinopleBlock) w.conf.Genesis.Config.ConstantinopleBlock = w.readDefaultBigInt(w.conf.Genesis.Config.ConstantinopleBlock) out, _ := json.MarshalIndent(w.conf.Genesis.Config, "", " ") @@ -177,18 +229,38 @@ func (w *wizard) manageGenesis() { case "2": // Save whatever genesis configuration we currently have fmt.Println() - fmt.Printf("Which base filename to save the genesis specifications into? (default = %s)\n", w.network) - out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") - basename := w.readDefaultString(fmt.Sprintf("%s.json", w.network)) + fmt.Printf("Which folder to save the genesis specs into? (default = current)\n") + fmt.Printf(" Will create %s.json, %s-aleth.json, %s-harmony.json, %s-parity.json\n", w.network, w.network, w.network, w.network) - gethJson := fmt.Sprintf("%s.json", basename) + folder := w.readDefaultString(".") + if err := os.MkdirAll(folder, 0755); err != nil { + log.Error("Failed to create spec folder", "folder", folder, "err", err) + return + } + out, _ := json.MarshalIndent(w.conf.Genesis, "", " ") + + // Export the native genesis spec used by puppeth and Geth + gethJson := filepath.Join(folder, fmt.Sprintf("%s.json", w.network)) if err := ioutil.WriteFile((gethJson), out, 0644); err != nil { log.Error("Failed to save genesis file", "err", err) + return } - log.Info("Saved geth genesis as %v", gethJson) - if err := convertGenesis(w.conf.Genesis, basename, w.network, w.conf.bootnodes); err != nil { - log.Error("Conversion failed", "err", err) + log.Info("Saved native genesis chain spec", "path", gethJson) + + // Export the genesis spec used by Aleth (formerly C++ Ethereum) + if spec, err := newAlethGenesisSpec(w.network, w.conf.Genesis); err != nil { + log.Error("Failed to create Aleth chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "aleth", spec) } + // Export the genesis spec used by Parity + if spec, err := newParityChainSpec(w.network, w.conf.Genesis, []string{}); err != nil { + log.Error("Failed to create Parity chain spec", "err", err) + } else { + saveGenesis(folder, w.network, "parity", spec) + } + // Export the genesis spec used by Harmony (formerly EthereumJ + saveGenesis(folder, w.network, "harmony", w.conf.Genesis) case "3": // Make sure we don't have any services running @@ -202,29 +274,18 @@ func (w *wizard) manageGenesis() { w.conf.flush() default: log.Error("That's not something I can do") + return } } -func saveGenesis(basename, client string, spec interface{}) { - filename := fmt.Sprintf("%s-%s.json", basename, client) +// saveGenesis JSON encodes an arbitrary genesis spec into a pre-defined file. +func saveGenesis(folder, network, client string, spec interface{}) { + path := filepath.Join(folder, fmt.Sprintf("%s-%s.json", network, client)) + out, _ := json.Marshal(spec) - if err := ioutil.WriteFile(filename, out, 0644); err != nil { - log.Error("failed to save genesis file", "client", client, "err", err) + if err := ioutil.WriteFile(path, out, 0644); err != nil { + log.Error("Failed to save genesis file", "client", client, "err", err) + return } - log.Info("saved chainspec", "client", client, "filename", filename) -} - -func convertGenesis(genesis *core.Genesis, basename string, network string, bootnodes []string) error { - if spec, err := newAlethGenesisSpec(network, genesis); err == nil { - saveGenesis(basename, "aleth", spec) - } else { - log.Error("failed to create chain spec", "client", "aleth", "err", err) - } - if spec, err := newParityChainSpec(network, genesis, []string{}); err == nil { - saveGenesis(basename, "parity", spec) - } else { - log.Error("failed to create chain spec", "client", "parity", "err", err) - } - saveGenesis(basename, "harmony", genesis) - return nil + log.Info("Saved genesis chain spec", "client", client, "path", path) } diff --git a/cmd/puppeth/wizard_intro.go b/cmd/puppeth/wizard_intro.go index 3db9a1087e..75fb04b76f 100644 --- a/cmd/puppeth/wizard_intro.go +++ b/cmd/puppeth/wizard_intro.go @@ -131,7 +131,20 @@ func (w *wizard) run() { case choice == "2": if w.conf.Genesis == nil { - w.makeGenesis() + fmt.Println() + fmt.Println("What would you like to do? (default = create)") + fmt.Println(" 1. Create new genesis from scratch") + fmt.Println(" 2. Import already existing genesis") + + choice := w.read() + switch { + case choice == "" || choice == "1": + w.makeGenesis() + case choice == "2": + w.importGenesis() + default: + log.Error("That's not something I can do") + } } else { w.manageGenesis() } @@ -149,7 +162,6 @@ func (w *wizard) run() { } else { w.manageComponents() } - default: log.Error("That's not something I can do") } diff --git a/cmd/puppeth/wizard_nginx.go b/cmd/puppeth/wizard_nginx.go index 4eeae93a0b..8397b7fd57 100644 --- a/cmd/puppeth/wizard_nginx.go +++ b/cmd/puppeth/wizard_nginx.go @@ -41,12 +41,12 @@ func (w *wizard) ensureVirtualHost(client *sshClient, port int, def string) (str // Reverse proxy is not running, offer to deploy a new one fmt.Println() fmt.Println("Allow sharing the port with other services (y/n)? (default = yes)") - if w.readDefaultString("y") == "y" { + if w.readDefaultYesNo(true) { nocache := false if proxy != nil { fmt.Println() fmt.Printf("Should the reverse-proxy be rebuilt from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNginx(client, w.network, port, nocache); err != nil { log.Error("Failed to deploy reverse-proxy", "err", err) diff --git a/cmd/puppeth/wizard_node.go b/cmd/puppeth/wizard_node.go index 49b10a023a..e37297f6d6 100644 --- a/cmd/puppeth/wizard_node.go +++ b/cmd/puppeth/wizard_node.go @@ -126,7 +126,7 @@ func (w *wizard) deployNode(boot bool) { } else { fmt.Println() fmt.Printf("Reuse previous (%s) signing account (y/n)? (default = yes)\n", key.Address.Hex()) - if w.readDefaultString("y") != "y" { + if !w.readDefaultYesNo(true) { infos.keyJSON, infos.keyPass = "", "" } } @@ -165,7 +165,7 @@ func (w *wizard) deployNode(boot bool) { if existed { fmt.Println() fmt.Printf("Should the node be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployNode(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy Ethereum node container", "err", err) diff --git a/cmd/puppeth/wizard_wallet.go b/cmd/puppeth/wizard_wallet.go index 7624d11e20..ca1ea5bd25 100644 --- a/cmd/puppeth/wizard_wallet.go +++ b/cmd/puppeth/wizard_wallet.go @@ -96,7 +96,7 @@ func (w *wizard) deployWallet() { if existed { fmt.Println() fmt.Printf("Should the wallet be built from scratch (y/n)? (default = no)\n") - nocache = w.readDefaultString("n") != "n" + nocache = w.readDefaultYesNo(false) } if out, err := deployWallet(client, w.network, w.conf.bootnodes, infos, nocache); err != nil { log.Error("Failed to deploy wallet container", "err", err) From 92639b676aaa7f262d19081256c43cdb78aa0efd Mon Sep 17 00:00:00 2001 From: Vedhavyas Singareddi Date: Tue, 4 Dec 2018 15:27:55 +0100 Subject: [PATCH 28/68] Add packing for dynamic array and slice types (#18051) * added tests for new abi encoding features (#4) * added tests from bytes32[][] and string[] * added offset to other types * formatting * Abi/dynamic types (#5) * Revert "Abi/dynamic types (#5)" (#6) This reverts commit dabca31d797623d43bd780721fc0ad461d24be53. * Abi/dynamic types (#7) * some cleanup * Apply suggestions from code review apply suggestions Co-Authored-By: vedhavyas * added better formatting (#8) * review chnages * better comments --- accounts/abi/argument.go | 19 +++++------- accounts/abi/pack_test.go | 62 ++++++++++++++++++++++++++++++++++++++- accounts/abi/type.go | 58 ++++++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index 93b513c346..f544c80dbb 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -243,11 +243,7 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { // input offset is the bytes offset for packed output inputOffset := 0 for _, abiArg := range abiArgs { - if abiArg.Type.T == ArrayTy { - inputOffset += 32 * abiArg.Type.Size - } else { - inputOffset += 32 - } + inputOffset += getDynamicTypeOffset(abiArg.Type) } var ret []byte for i, a := range args { @@ -257,14 +253,13 @@ func (arguments Arguments) Pack(args ...interface{}) ([]byte, error) { if err != nil { return nil, err } - // check for a slice type (string, bytes, slice) - if input.Type.requiresLengthPrefix() { - // calculate the offset - offset := inputOffset + len(variableInput) + // check for dynamic types + if isDynamicType(input.Type) { // set the offset - ret = append(ret, packNum(reflect.ValueOf(offset))...) - // Append the packed output to the variable input. The variable input - // will be appended at the end of the input. + ret = append(ret, packNum(reflect.ValueOf(inputOffset))...) + // calculate next offset + inputOffset += len(packed) + // append to variable input variableInput = append(variableInput, packed...) } else { // append the packed value to the input diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 58a5b7a581..ddd2b7362d 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -324,6 +324,66 @@ func TestPack(t *testing.T) { "foobar", common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000006666f6f6261720000000000000000000000000000000000000000000000000000"), }, + { + "string[]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset 128 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "string[2]", + []string{"hello", "foobar"}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset to i = 0 + "0000000000000000000000000000000000000000000000000000000000000080" + // offset to i = 1 + "0000000000000000000000000000000000000000000000000000000000000005" + // len(str[0]) = 5 + "68656c6c6f000000000000000000000000000000000000000000000000000000" + // str[0] + "0000000000000000000000000000000000000000000000000000000000000006" + // len(str[1]) = 6 + "666f6f6261720000000000000000000000000000000000000000000000000000"), // str[1] + }, + { + "bytes32[][]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000002" + // len(array) = 2 + "0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[][2]", + [][]common.Hash{{{1}, {2}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000040" + // offset 64 to i = 0 + "00000000000000000000000000000000000000000000000000000000000000a0" + // offset 160 to i = 1 + "0000000000000000000000000000000000000000000000000000000000000002" + // len(array[0]) = 2 + "0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0000000000000000000000000000000000000000000000000000000000000003" + // len(array[1]) = 3 + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, + + { + "bytes32[3][2]", + [][]common.Hash{{{1}, {2}, {3}}, {{3}, {4}, {5}}}, + common.Hex2Bytes("0100000000000000000000000000000000000000000000000000000000000000" + // array[0][0] + "0200000000000000000000000000000000000000000000000000000000000000" + // array[0][1] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[0][2] + "0300000000000000000000000000000000000000000000000000000000000000" + // array[1][0] + "0400000000000000000000000000000000000000000000000000000000000000" + // array[1][1] + "0500000000000000000000000000000000000000000000000000000000000000"), // array[1][2] + }, } { typ, err := NewType(test.typ) if err != nil { @@ -336,7 +396,7 @@ func TestPack(t *testing.T) { } if !bytes.Equal(output, test.output) { - t.Errorf("%d failed. Expected bytes: '%x' Got: '%x'", i, test.output, output) + t.Errorf("input %d for typ: %v failed. Expected bytes: '%x' Got: '%x'", i, typ.String(), test.output, output) } } } diff --git a/accounts/abi/type.go b/accounts/abi/type.go index dce89d2b4e..6bfaabf5a5 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -183,23 +183,39 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { return nil, err } - if t.T == SliceTy || t.T == ArrayTy { - var packed []byte + switch t.T { + case SliceTy, ArrayTy: + var ret []byte + if t.requiresLengthPrefix() { + // append length + ret = append(ret, packNum(reflect.ValueOf(v.Len()))...) + } + + // calculate offset if any + offset := 0 + offsetReq := isDynamicType(*t.Elem) + if offsetReq { + offset = getDynamicTypeOffset(*t.Elem) * v.Len() + } + var tail []byte for i := 0; i < v.Len(); i++ { val, err := t.Elem.pack(v.Index(i)) if err != nil { return nil, err } - packed = append(packed, val...) - } - if t.T == SliceTy { - return packBytesSlice(packed, v.Len()), nil - } else if t.T == ArrayTy { - return packed, nil + if !offsetReq { + ret = append(ret, val...) + continue + } + ret = append(ret, packNum(reflect.ValueOf(offset))...) + offset += len(val) + tail = append(tail, val...) } + return append(ret, tail...), nil + default: + return packElement(t, v), nil } - return packElement(t, v), nil } // requireLengthPrefix returns whether the type requires any sort of length @@ -207,3 +223,27 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { func (t Type) requiresLengthPrefix() bool { return t.T == StringTy || t.T == BytesTy || t.T == SliceTy } + +// isDynamicType returns true if the type is dynamic. +// StringTy, BytesTy, and SliceTy(irrespective of slice element type) are dynamic types +// ArrayTy is considered dynamic if and only if the Array element is a dynamic type. +// This function recursively checks the type for slice and array elements. +func isDynamicType(t Type) bool { + // dynamic types + // array is also a dynamic type if the array type is dynamic + return t.T == StringTy || t.T == BytesTy || t.T == SliceTy || (t.T == ArrayTy && isDynamicType(*t.Elem)) +} + +// getDynamicTypeOffset returns the offset for the type. +// See `isDynamicType` to know which types are considered dynamic. +// If the type t is an array and element type is not a dynamic type, then we consider it a static type and +// return 32 * size of array since length prefix is not required. +// If t is a dynamic type or element type(for slices and arrays) is dynamic, then we simply return 32 as offset. +func getDynamicTypeOffset(t Type) int { + // if it is an array and there are no dynamic types + // then the array is static type + if t.T == ArrayTy && !isDynamicType(*t.Elem) { + return 32 * t.Size + } + return 32 +} From b98d2e9a1c11f797502a3afa711e470a41cbc91d Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 4 Dec 2018 12:29:51 -0500 Subject: [PATCH 29/68] swarm/network/stream: Debug log instead of Warn for retrieval failure (#18246) --- swarm/network/stream/delivery.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/swarm/network/stream/delivery.go b/swarm/network/stream/delivery.go index a5c9fa7e86..c73298d9aa 100644 --- a/swarm/network/stream/delivery.go +++ b/swarm/network/stream/delivery.go @@ -39,6 +39,7 @@ const ( var ( processReceivedChunksCount = metrics.NewRegisteredCounter("network.stream.received_chunks.count", nil) handleRetrieveRequestMsgCount = metrics.NewRegisteredCounter("network.stream.handle_retrieve_request_msg.count", nil) + retrieveChunkFail = metrics.NewRegisteredCounter("network.stream.retrieve_chunks_fail.count", nil) requestFromPeersCount = metrics.NewRegisteredCounter("network.stream.request_from_peers.count", nil) requestFromPeersEachCount = metrics.NewRegisteredCounter("network.stream.request_from_peers_each.count", nil) @@ -169,7 +170,8 @@ func (d *Delivery) handleRetrieveRequestMsg(ctx context.Context, sp *Peer, req * go func() { chunk, err := d.chunkStore.Get(ctx, req.Addr) if err != nil { - log.Warn("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) + retrieveChunkFail.Inc(1) + log.Debug("ChunkStore.Get can not retrieve chunk", "peer", sp.ID().String(), "addr", req.Addr, "hopcount", req.HopCount, "err", err) return } if req.SkipCheck { From 3ac633ba8423851a85cb909a1f1f0f3c0d2c5351 Mon Sep 17 00:00:00 2001 From: Elad Date: Wed, 5 Dec 2018 15:50:55 +0530 Subject: [PATCH 30/68] swarm/api/http: add resetting timer metrics to requests (#18249) --- swarm/api/http/middleware.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index f5f70138b3..115a008566 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -5,6 +5,7 @@ import ( "net/http" "runtime/debug" "strings" + "time" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" @@ -73,9 +74,13 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + startTime := time.Now() + defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) }) } From de39513ced05f120b30b0daa49003982cc6b2426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 6 Dec 2018 14:34:49 +0100 Subject: [PATCH 31/68] core, internal, eth, miner, les: Take VM config from BlockChain (#17955) Until this commit, when sending an RPC request that called `NewEVM`, a blank `vm.Config` would be taken so as to set some options, based on the default configuration. If some extra configuration switches were passed to the blockchain, those would be ignored. This PR adds a function to get the config from the blockchain, and this is what is now used for RPC calls. Some subsequent changes need to be made, see https://github.com/ethereum/go-ethereum/pull/17955#pullrequestreview-182237244 for the details of the discussion. --- core/blockchain.go | 5 +++++ eth/api_backend.go | 4 ++-- internal/ethapi/api.go | 8 ++++---- internal/ethapi/backend.go | 2 +- les/api_backend.go | 4 ++-- miner/worker.go | 3 +-- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index a672c0ab64..c29063a73b 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -210,6 +210,11 @@ func (bc *BlockChain) getProcInterrupt() bool { return atomic.LoadInt32(&bc.procInterrupt) == 1 } +// GetVMConfig returns the block chain VM config. +func (bc *BlockChain) GetVMConfig() *vm.Config { + return &bc.vmConfig +} + // loadLastState loads the last known chain state from the database. This method // assumes that the chain manager mutex is held. func (bc *BlockChain) loadLastState() error { diff --git a/eth/api_backend.go b/eth/api_backend.go index 8748d444fa..a48815e0db 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -125,12 +125,12 @@ func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(blockHash) } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) - return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), vmError, nil + return vm.NewEVM(context, state, b.eth.chainConfig, *b.eth.blockchain.GetVMConfig()), vmError, nil } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 43a33e9921..656555b3ba 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -683,7 +683,7 @@ type CallArgs struct { Data hexutil.Bytes `json:"data"` } -func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, vmCfg vm.Config, timeout time.Duration) ([]byte, uint64, bool, error) { +func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber, timeout time.Duration) ([]byte, uint64, bool, error) { defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) state, header, err := s.b.StateAndHeaderByNumber(ctx, blockNr) @@ -724,7 +724,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr defer cancel() // Get a new instance of the EVM. - evm, vmError, err := s.b.GetEVM(ctx, msg, state, header, vmCfg) + evm, vmError, err := s.b.GetEVM(ctx, msg, state, header) if err != nil { return nil, 0, false, err } @@ -748,7 +748,7 @@ func (s *PublicBlockChainAPI) doCall(ctx context.Context, args CallArgs, blockNr // Call executes the given transaction on the state for the given block number. // It doesn't make and changes in the state/blockchain and is useful to execute and retrieve values. func (s *PublicBlockChainAPI) Call(ctx context.Context, args CallArgs, blockNr rpc.BlockNumber) (hexutil.Bytes, error) { - result, _, _, err := s.doCall(ctx, args, blockNr, vm.Config{}, 5*time.Second) + result, _, _, err := s.doCall(ctx, args, blockNr, 5*time.Second) return (hexutil.Bytes)(result), err } @@ -777,7 +777,7 @@ func (s *PublicBlockChainAPI) EstimateGas(ctx context.Context, args CallArgs) (h executable := func(gas uint64) bool { args.Gas = hexutil.Uint64(gas) - _, _, failed, err := s.doCall(ctx, args, rpc.PendingBlockNumber, vm.Config{}, 0) + _, _, failed, err := s.doCall(ctx, args, rpc.PendingBlockNumber, 0) if err != nil || failed { return false } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index c9ffe230c6..e23ee03b15 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -53,7 +53,7 @@ type Backend interface { GetBlock(ctx context.Context, blockHash common.Hash) (*types.Block, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) GetTd(blockHash common.Hash) *big.Int - GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) + GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/les/api_backend.go b/les/api_backend.go index aa748a4ea1..7531396235 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -105,10 +105,10 @@ func (b *LesApiBackend) GetTd(hash common.Hash) *big.Int { return b.eth.blockchain.GetTdByHash(hash) } -func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmCfg vm.Config) (*vm.EVM, func() error, error) { +func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { state.SetBalance(msg.From(), math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) - return vm.NewEVM(context, state, b.eth.chainConfig, vmCfg), state.Error, nil + return vm.NewEVM(context, state, b.eth.chainConfig, vm.Config{}), state.Error, nil } func (b *LesApiBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { diff --git a/miner/worker.go b/miner/worker.go index 8579c5c84b..48473796bc 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -692,7 +691,7 @@ func (w *worker) updateSnapshot() { func (w *worker) commitTransaction(tx *types.Transaction, coinbase common.Address) ([]*types.Log, error) { snap := w.current.state.Snapshot() - receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, vm.Config{}) + receipt, _, err := core.ApplyTransaction(w.config, w.chain, &coinbase, w.current.gasPool, w.current.state, w.current.header, tx, &w.current.header.GasUsed, *w.chain.GetVMConfig()) if err != nil { w.current.state.RevertToSnapshot(snap) return nil, err From 661809714e35f69dea23f713dd1e65cff523344c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jano=C5=A1=20Gulja=C5=A1?= Date: Fri, 7 Dec 2018 06:51:40 +0100 Subject: [PATCH 32/68] swarm: snapshot load improvement (#18220) * swarm/network: Hive - do not notify peer if discovery is disabled * p2p/simulations: validate all connections on loading a snapshot * p2p/simulations: track all connections in on snapshot loading * p2p/simulations: add snapshotLoadTimeout variable * p2p/simulations: ignore control events in snapshot load * p2p/simulations: simplify event loop synchronization * p2p/simulations: return already connected error from Load function * p2p/simulations: log warning on snapshot loading disconnection --- p2p/simulations/network.go | 75 +++++++++++++++++++++++ swarm/network/hive.go | 2 +- swarm/network/simulation/kademlia_test.go | 1 - 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index 92ccfde817..ab9f582c52 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -20,6 +20,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "sync" "time" @@ -705,8 +706,11 @@ func (net *Network) snapshot(addServices []string, removeServices []string) (*Sn return snap, nil } +var snapshotLoadTimeout = 120 * time.Second + // Load loads a network snapshot func (net *Network) Load(snap *Snapshot) error { + // Start nodes. for _, n := range snap.Nodes { if _, err := net.NewNodeWithConfig(n.Node.Config); err != nil { return err @@ -718,6 +722,69 @@ func (net *Network) Load(snap *Snapshot) error { return err } } + + // Prepare connection events counter. + allConnected := make(chan struct{}) // closed when all connections are established + done := make(chan struct{}) // ensures that the event loop goroutine is terminated + defer close(done) + + // Subscribe to event channel. + // It needs to be done outside of the event loop goroutine (created below) + // to ensure that the event channel is blocking before connect calls are made. + events := make(chan *Event) + sub := net.Events().Subscribe(events) + defer sub.Unsubscribe() + + go func() { + // Expected number of connections. + total := len(snap.Conns) + // Set of all established connections from the snapshot, not other connections. + // Key array element 0 is the connection One field value, and element 1 connection Other field. + connections := make(map[[2]enode.ID]struct{}, total) + + for { + select { + case e := <-events: + // Ignore control events as they do not represent + // connect or disconnect (Up) state change. + if e.Control { + continue + } + // Detect only connection events. + if e.Type != EventTypeConn { + continue + } + connection := [2]enode.ID{e.Conn.One, e.Conn.Other} + // Nodes are still not connected or have been disconnected. + if !e.Conn.Up { + // Delete the connection from the set of established connections. + // This will prevent false positive in case disconnections happen. + delete(connections, connection) + log.Warn("load snapshot: unexpected disconnection", "one", e.Conn.One, "other", e.Conn.Other) + continue + } + // Check that the connection is from the snapshot. + for _, conn := range snap.Conns { + if conn.One == e.Conn.One && conn.Other == e.Conn.Other { + // Add the connection to the set of established connections. + connections[connection] = struct{}{} + if len(connections) == total { + // Signal that all nodes are connected. + close(allConnected) + return + } + + break + } + } + case <-done: + // Load function returned, terminate this goroutine. + return + } + } + }() + + // Start connecting. for _, conn := range snap.Conns { if !net.GetNode(conn.One).Up || !net.GetNode(conn.Other).Up { @@ -729,6 +796,14 @@ func (net *Network) Load(snap *Snapshot) error { return err } } + + select { + // Wait until all connections from the snapshot are established. + case <-allConnected: + // Make sure that we do not wait forever. + case <-time.After(snapshotLoadTimeout): + return errors.New("snapshot connections not established") + } return nil } diff --git a/swarm/network/hive.go b/swarm/network/hive.go index 1aa1ae42a2..ebef545929 100644 --- a/swarm/network/hive.go +++ b/swarm/network/hive.go @@ -165,8 +165,8 @@ func (h *Hive) Run(p *BzzPeer) error { // otherwise just send depth to new peer dp.NotifyDepth(depth) } + NotifyPeer(p.BzzAddr, h.Kademlia) } - NotifyPeer(p.BzzAddr, h.Kademlia) defer h.Off(dp) return dp.Run(dp.HandleMsg) } diff --git a/swarm/network/simulation/kademlia_test.go b/swarm/network/simulation/kademlia_test.go index 024830315a..f02b0e5417 100644 --- a/swarm/network/simulation/kademlia_test.go +++ b/swarm/network/simulation/kademlia_test.go @@ -33,7 +33,6 @@ func TestWaitTillHealthy(t *testing.T) { "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { addr := network.NewAddr(ctx.Config.Node()) hp := network.NewHiveParams() - hp.Discovery = false config := &network.BzzConfig{ OverlayAddr: addr.Over(), UnderlayAddr: addr.Under(), From f32790fb0508bb484a100d6e5c81bec2c89bebb1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 7 Dec 2018 14:43:27 +0100 Subject: [PATCH 33/68] node: warn when using deprecated config/resource files (#18199) --- node/config.go | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/node/config.go b/node/config.go index 8f10f4f613..7b32a59087 100644 --- a/node/config.go +++ b/node/config.go @@ -24,6 +24,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" @@ -152,6 +153,10 @@ type Config struct { // Logger is a custom logger to use with the p2p.Server. Logger log.Logger `toml:",omitempty"` + + staticNodesWarning bool + trustedNodesWarning bool + oldGethResourceWarning bool } // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into @@ -263,8 +268,8 @@ var isOldGethResource = map[string]bool{ "chaindata": true, "nodes": true, "nodekey": true, - "static-nodes.json": true, - "trusted-nodes.json": true, + "static-nodes.json": false, // no warning for these because they have their + "trusted-nodes.json": false, // own separate warning. } // ResolvePath resolves path in the instance directory. @@ -277,13 +282,15 @@ func (c *Config) ResolvePath(path string) string { } // Backwards-compatibility: ensure that data directory files created // by geth 1.4 are used if they exist. - if c.name() == "geth" && isOldGethResource[path] { + if warn, isOld := isOldGethResource[path]; isOld { oldpath := "" - if c.Name == "geth" { + if c.name() == "geth" { oldpath = filepath.Join(c.DataDir, path) } if oldpath != "" && common.FileExist(oldpath) { - // TODO: print warning + if warn { + c.warnOnce(&c.oldGethResourceWarning, "Using deprecated resource file %s, please move this file to the 'geth' subdirectory of datadir.", oldpath) + } return oldpath } } @@ -337,17 +344,17 @@ func (c *Config) NodeKey() *ecdsa.PrivateKey { // StaticNodes returns a list of node enode URLs configured as static nodes. func (c *Config) StaticNodes() []*enode.Node { - return c.parsePersistentNodes(c.ResolvePath(datadirStaticNodes)) + return c.parsePersistentNodes(&c.staticNodesWarning, c.ResolvePath(datadirStaticNodes)) } // TrustedNodes returns a list of node enode URLs configured as trusted nodes. func (c *Config) TrustedNodes() []*enode.Node { - return c.parsePersistentNodes(c.ResolvePath(datadirTrustedNodes)) + return c.parsePersistentNodes(&c.trustedNodesWarning, c.ResolvePath(datadirTrustedNodes)) } // parsePersistentNodes parses a list of discovery node URLs loaded from a .json // file from within the data directory. -func (c *Config) parsePersistentNodes(path string) []*enode.Node { +func (c *Config) parsePersistentNodes(w *bool, path string) []*enode.Node { // Short circuit if no node config is present if c.DataDir == "" { return nil @@ -355,10 +362,12 @@ func (c *Config) parsePersistentNodes(path string) []*enode.Node { if _, err := os.Stat(path); err != nil { return nil } + c.warnOnce(w, "Found deprecated node list file %s, please use the TOML config file instead.", path) + // Load the nodes from the config file. var nodelist []string if err := common.LoadJSON(path, &nodelist); err != nil { - log.Error(fmt.Sprintf("Can't load node file %s: %v", path, err)) + log.Error(fmt.Sprintf("Can't load node list file: %v", err)) return nil } // Interpret the list as a discovery node array @@ -440,3 +449,20 @@ func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { } return accounts.NewManager(backends...), ephemeral, nil } + +var warnLock sync.Mutex + +func (c *Config) warnOnce(w *bool, format string, args ...interface{}) { + warnLock.Lock() + defer warnLock.Unlock() + + if *w { + return + } + l := c.Logger + if l == nil { + l = log.Root() + } + l.Warn(fmt.Sprintf(format, args...)) + *w = true +} From c1d462ee5d3a22746e90531e06139b703f0e040f Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Mon, 10 Dec 2018 10:16:19 +0100 Subject: [PATCH 34/68] cmd/puppeth: fix rogue quote in alethGenesisSpec JSON (#18262) --- cmd/puppeth/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/genesis.go b/cmd/puppeth/genesis.go index 83212a126b..1025dfe826 100644 --- a/cmd/puppeth/genesis.go +++ b/cmd/puppeth/genesis.go @@ -54,7 +54,7 @@ type alethGenesisSpec struct { BlockReward *hexutil.Big `json:"blockReward"` NetworkID hexutil.Uint64 `json:"networkID"` ChainID hexutil.Uint64 `json:"chainID"` - AllowFutureBlocks bool `json:"allowFutureBlocks""` + AllowFutureBlocks bool `json:"allowFutureBlocks"` } `json:"params"` Genesis struct { From dfa16a3e4e0e0b5b20bfda7b7e89ebd07ea0a1a5 Mon Sep 17 00:00:00 2001 From: Max Sistemich Date: Mon, 10 Dec 2018 10:17:31 +0100 Subject: [PATCH 35/68] eth/tracers: fixed incorrect storage from prestate_tracer (#18253) * eth: fixed incorrect storage from prestate_tracer * eth/tracers: updated assets.go --- eth/tracers/internal/tracers/assets.go | 26 +++++++++---------- .../internal/tracers/prestate_tracer.js | 5 +--- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 04dd6fe89f..addd328825 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -1,14 +1,14 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: -// 4byte_tracer.js -// bigram_tracer.js -// call_tracer.js -// evmdis_tracer.js -// noop_tracer.js -// opcount_tracer.js -// prestate_tracer.js -// trigram_tracer.js -// unigram_tracer.js +// 4byte_tracer.js (2.933kB) +// bigram_tracer.js (1.712kB) +// call_tracer.js (8.596kB) +// evmdis_tracer.js (4.194kB) +// noop_tracer.js (1.271kB) +// opcount_tracer.js (1.372kB) +// prestate_tracer.js (3.892kB) +// trigram_tracer.js (1.788kB) +// unigram_tracer.js (1.51kB) package tracers @@ -28,7 +28,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } var buf bytes.Buffer @@ -36,7 +36,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } if clErr != nil { return nil, err @@ -197,7 +197,7 @@ func opcount_tracerJs() (*asset, error) { return a, nil } -var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x57\xdd\x6f\x1b\xb9\x11\x7f\xde\xfd\x2b\xa6\x7e\x91\x84\x53\x56\xce\x15\xb8\x02\x72\x5d\x60\xa3\x28\x89\x00\x9d\x6d\x48\x4a\x5d\xf7\x70\x0f\x5c\x72\x76\xc5\x13\x45\x2e\x48\xae\x3e\x10\xf8\x7f\x2f\x86\xfb\x21\xcb\x67\x27\x6e\xeb\x27\x2f\x39\xfc\xcd\xf7\x6f\x46\xa3\x11\x4c\x4c\x79\xb4\xb2\x58\x7b\xf8\xf9\xf2\xfd\xdf\x60\xb5\x46\x28\xcc\x3b\xf4\x6b\xb4\x58\x6d\x21\xad\xfc\xda\x58\x17\x8f\x46\xb0\x5a\x4b\x07\xb9\x54\x08\xd2\x41\xc9\xac\x07\x93\x83\x7f\x26\xaf\x64\x66\x99\x3d\x26\xf1\x68\x54\xbf\x79\xf1\x9a\x10\x72\x8b\x08\xce\xe4\x7e\xcf\x2c\x8e\xe1\x68\x2a\xe0\x4c\x83\x45\x21\x9d\xb7\x32\xab\x3c\x82\xf4\xc0\xb4\x18\x19\x0b\x5b\x23\x64\x7e\x24\x48\xe9\xa1\xd2\x02\x6d\x50\xed\xd1\x6e\x5d\x6b\xc7\xe7\x9b\xaf\x30\x47\xe7\xd0\xc2\x67\xd4\x68\x99\x82\xbb\x2a\x53\x92\xc3\x5c\x72\xd4\x0e\x81\x39\x28\xe9\xc4\xad\x51\x40\x16\xe0\xe8\xe1\x27\x32\x65\xd9\x98\x02\x9f\x4c\xa5\x05\xf3\xd2\xe8\x21\xa0\x24\xcb\x61\x87\xd6\x49\xa3\xe1\xaf\xad\xaa\x06\x70\x08\xc6\x12\x48\x9f\x79\x72\xc0\x82\x29\xe9\xdd\x00\x98\x3e\x82\x62\xfe\xf4\xf4\x0d\x01\x39\xf9\x2d\x40\xea\xa0\x66\x6d\x4a\x04\xbf\x66\x9e\xbc\xde\x4b\xa5\x20\x43\xa8\x1c\xe6\x95\x1a\x12\x5a\x56\x79\xb8\x9f\xad\xbe\xdc\x7e\x5d\x41\x7a\xf3\x00\xf7\xe9\x62\x91\xde\xac\x1e\xae\x60\x2f\xfd\xda\x54\x1e\x70\x87\x35\x94\xdc\x96\x4a\xa2\x80\x3d\xb3\x96\x69\x7f\x04\x93\x13\xc2\xaf\xd3\xc5\xe4\x4b\x7a\xb3\x4a\x3f\xcc\xe6\xb3\xd5\x03\x18\x0b\x9f\x66\xab\x9b\xe9\x72\x09\x9f\x6e\x17\x90\xc2\x5d\xba\x58\xcd\x26\x5f\xe7\xe9\x02\xee\xbe\x2e\xee\x6e\x97\xd3\x04\x96\x48\x56\x21\xbd\xff\x71\xcc\xf3\x90\x3d\x8b\x20\xd0\x33\xa9\x5c\x1b\x89\x07\x53\x81\x5b\x9b\x4a\x09\x58\xb3\x1d\x82\x45\x8e\x72\x87\x02\x18\x70\x53\x1e\xdf\x9c\x54\xc2\x62\xca\xe8\x22\xf8\xfc\x6a\x41\xc2\x2c\x07\x6d\xfc\x10\x1c\x22\xfc\x7d\xed\x7d\x39\x1e\x8d\xf6\xfb\x7d\x52\xe8\x2a\x31\xb6\x18\xa9\x1a\xce\x8d\xfe\x91\xc4\x84\x59\x5a\x74\x9e\x79\x5c\x59\xc6\xd1\x82\xa9\x7c\x59\x79\x07\xae\xca\x73\xc9\x25\x6a\x0f\x52\xe7\xc6\x6e\x43\xa5\x80\x37\xc0\x2d\x32\x8f\xc0\x40\x19\xce\x14\xe0\x01\x79\x15\xee\xea\x48\x87\x72\xb5\x4c\x3b\xc6\xc3\x69\x6e\xcd\x96\x7c\xad\x9c\xa7\x7f\x9c\xc3\x6d\xa6\x50\x40\x81\x1a\x9d\x74\x90\x29\xc3\x37\x49\xfc\x2d\x8e\x9e\x18\x43\x75\x12\x3c\x6c\x84\x42\x6d\xec\xb1\x67\x11\xb2\x4a\x2a\x21\x75\x91\xc4\x51\x2b\x3d\x06\x5d\x29\x35\x8c\x03\x84\x32\x66\x53\x95\x29\xe7\xa6\x0a\xb6\xff\x81\xdc\xd7\x60\xae\x44\x2e\x73\x2a\x0e\xd6\xdd\x7a\x13\xae\x3a\xbd\x26\x23\xf9\x24\x8e\xce\x60\xc6\x90\x57\x3a\xb8\xd3\x67\x42\xd8\x21\x88\x6c\xf0\x2d\x8e\xa2\x1d\xb3\x84\x05\xd7\xe0\xcd\x17\x3c\x84\xcb\xc1\x55\x1c\x45\x32\x87\xbe\x5f\x4b\x97\xb4\xc0\xbf\x31\xce\x7f\x87\xeb\xeb\xeb\xd0\xd4\xb9\xd4\x28\x06\x40\x10\xd1\x4b\x62\xf5\x4d\x94\x31\xc5\x34\xc7\x31\xf4\x2e\x0f\x3d\xf8\x09\x44\x96\x14\xe8\x3f\xd4\xa7\xb5\xb2\xc4\x9b\xa5\xb7\x52\x17\xfd\xf7\xbf\x0c\x86\xe1\x95\x36\xe1\x0d\x34\xe2\x37\xa6\x13\xae\xef\xb9\x11\xe1\xba\xb1\xb9\x96\x9a\x18\xd1\x08\x35\x52\xce\x1b\xcb\x0a\x1c\xc3\xb7\x47\xfa\x7e\x24\xaf\x1e\xe3\xe8\xf1\x2c\xca\xcb\x5a\xe8\x95\x28\x37\x10\x80\xda\xdb\xae\xce\x0b\x49\x9d\xfa\x34\x01\x01\xef\x7b\x49\x58\xb6\xa6\x3c\x4b\xc2\x06\x8f\x3f\xce\x04\x5d\x48\x71\xe8\x2e\x36\x78\x1c\x5c\xc5\xaf\xa6\x28\x69\x8c\xfe\x4d\x8a\xc3\xcb\xf9\x22\xc0\x1d\x53\x1d\x60\x1d\xbf\x25\x21\x9c\xec\x1a\x04\xdd\x41\x07\xc9\xfe\xe5\x1a\x2e\x2e\x0f\x97\xff\xe7\xdf\x45\x63\xc1\x0b\x25\xf3\xcc\xec\x37\x98\xf6\x78\x9e\x4f\x8b\xae\x52\x9e\xda\x4e\xea\x9d\xd9\x10\x81\xae\x29\x4f\x4a\x85\xd4\x98\x92\xaa\xc6\xd5\x0c\x96\x21\x6a\x90\x1e\x2d\x23\x0a\x37\x3b\xb4\x34\xbd\xc0\xa2\xaf\xac\x76\x5d\x3a\x73\xa9\x99\x6a\x81\x9b\xec\x7b\xcb\x78\xdd\xbb\xf5\xf9\x93\x9c\x72\x7f\x08\xd9\x0c\x3e\x8e\x46\x90\x7a\x20\x3f\xa1\x34\x52\xfb\x21\xec\x11\x34\xa2\x20\x02\x12\x28\x2a\xee\x03\x5e\x6f\xc7\x54\x85\xbd\x9a\x64\x88\xaa\xc3\x53\x53\xd1\x44\x7a\x42\x42\xc3\x60\xe0\xd6\xec\xc2\xa8\xcd\x18\xdf\x40\xd3\xf8\xc6\xca\x42\xea\xb8\x89\xe9\x59\xd3\x93\x45\x09\x01\x07\xb3\x42\xcd\x50\xee\xe9\xe4\x43\xc8\x7f\x26\x8b\x99\xf6\xcf\x8a\xa8\x8e\x7c\xfb\x74\xf0\x7b\xd2\x34\x71\xe2\x88\x78\xfb\x3f\x0f\x86\xf0\xfe\x97\xae\x32\xbd\x21\x28\xf8\x31\x98\x37\xaf\x43\xc5\xcf\x2b\xe2\xe5\x67\x41\x0d\x31\xc9\x4f\x41\x6b\xe2\xaa\x8c\xd2\x51\xfb\x19\xe2\x78\xce\x26\x57\xdf\xc1\x3d\xf7\xad\xc5\x6d\x42\x93\x30\x21\x5e\x07\xad\x53\xf4\x11\xb9\xc5\x2d\x4d\x17\xca\x02\x67\x4a\xa1\xed\x39\x08\xdc\x35\x6c\xca\x29\xe4\x0b\xb7\xa5\x3f\xb6\x33\xc7\x33\x5b\xa0\x77\x3f\x36\x2c\xe0\xbc\x7b\xd7\x52\x71\x08\xc5\xb1\x44\xb8\xbe\x86\xde\x64\x31\x4d\x57\xd3\x5e\xd3\x4c\xa3\x11\xdc\x63\xd8\xc8\x32\x25\x33\xa1\x8e\x20\x50\xa1\xc7\xda\x2e\xa3\x43\x88\x3a\x6a\x1a\xd2\x6a\x45\x4b\x0f\x1e\xa4\xf3\x52\x17\x50\x33\xd6\x9e\xe6\x7b\x03\x17\x7a\x84\xb3\xca\x51\xb5\x3e\x1b\x86\xde\xd0\x66\x63\x91\xf8\x8d\xe6\x50\x68\x37\xa6\x64\xb7\x09\xe5\xd2\x3a\x0f\xa5\x62\x1c\x13\xc2\xeb\x8c\x79\x3d\xbf\x0d\x33\x93\xea\x45\x68\xc1\x00\x74\x1a\xb4\x4c\xd1\xa0\x26\xf5\x0e\xfa\x2d\xc6\x20\x8e\x22\xdb\x4a\x3f\xc1\xbe\x3a\x51\x82\xf3\x58\x3e\x25\x04\x5a\x70\x70\x87\x44\xe5\x81\x0d\xea\xa1\x4c\xba\xfe\xf9\x6b\xb3\x05\xa0\x4b\xe2\x88\xde\x3d\xe9\x6b\x65\x8a\xf3\xbe\x16\x75\x58\x78\x65\x2d\xe5\xbf\x1b\x05\x39\xf5\xf8\x1f\x95\xf3\x14\x53\x4b\xe1\x69\xd8\xe2\x25\xb2\x0e\xd4\x4c\x53\x7f\xf0\xe7\x21\x4a\xf3\x33\xcc\x2b\x52\xd7\x4c\xcb\x7a\xab\x2c\x8d\x47\xed\x25\x53\xea\x48\x79\xd8\x5b\x5a\xa7\x68\x81\x1a\x82\x93\x24\x15\x18\x27\x88\x4a\xcd\x55\x25\xea\x32\x08\x75\xdc\xe0\xb9\x60\xf3\xf9\x1e\xb6\x45\xe7\x58\x81\x09\x55\x52\x2e\x0f\xcd\x26\xab\xa1\x57\x93\x5c\x7f\xd0\x4b\x3a\x23\xcf\x29\x46\x99\x22\x69\x8b\x8c\xb8\x3a\x15\xc2\xa2\x73\xfd\x41\xc3\x39\x5d\x66\xef\xd7\xa8\x29\xf8\xa0\x71\x0f\xdd\x8a\xc4\x38\xa7\x95\x51\x0c\x81\x09\x41\xd4\xf6\x6c\x9d\x89\xa3\xc8\xed\xa5\xe7\x6b\x08\x9a\x4c\x79\xea\xc5\x41\x53\xff\x9c\x39\x84\x8b\xe9\xbf\x56\x93\xdb\x8f\xd3\xc9\xed\xdd\xc3\xc5\x18\xce\xce\x96\xb3\x7f\x4f\xbb\xb3\x0f\xe9\x3c\xbd\x99\x4c\x2f\xc6\xa7\x39\x74\xee\x90\x37\xad\x0b\xa4\xd0\x79\xc6\x37\x49\x89\xb8\xe9\x5f\x9e\xf3\xc0\xc9\xc1\x28\xca\x2c\xb2\xcd\xd5\xc9\x98\xba\x41\x1b\x1d\x2d\xe5\xc2\x35\xbc\x1a\xac\xab\xd7\xad\x99\x34\xf2\xfd\x96\xc8\x4f\x2b\x51\xa0\x8a\xef\xda\x91\xce\xe7\x9d\xe7\xf4\x41\xe1\xe8\x0e\x3e\x4e\xe7\xd3\xcf\xe9\x6a\x7a\x26\xb5\x5c\xa5\xab\xd9\xa4\x3e\xfa\xaf\x43\xf4\xfe\xcd\x21\xea\x2d\x97\xab\xdb\xc5\xb4\x37\x6e\xbe\xe6\xb7\xe9\xc7\xde\x9f\x14\x36\x7b\xd3\xf7\x8a\xcc\x9b\x7b\x63\xc5\xff\x92\xab\x27\xbb\x43\xce\x5e\x5a\x1d\x02\x09\x71\x5f\x3d\xfb\x89\x00\x4c\xb7\xfc\x91\xd7\x3f\x93\xa2\xf0\xfe\x45\xc6\x78\x8c\x1f\xe3\xff\x04\x00\x00\xff\xff\xb5\x44\x89\xaf\xbc\x0f\x00\x00") +var _prestate_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\x57\xdf\x6f\x1a\x49\x12\x7e\x9e\xf9\x2b\x4a\xfb\x02\x68\xc9\x90\xec\x49\x7b\x12\x3e\x9f\x34\x21\x24\x41\x62\x6d\x0b\xc8\xf9\x7c\xab\x7d\xe8\xe9\xae\x19\x7a\x69\xba\x47\xfd\x03\x8c\x22\xff\xef\xa7\xea\x99\x01\xc3\xda\x49\xee\xde\xcc\x74\xf5\x57\x55\x5f\x55\x7d\x5d\x1e\x8d\x60\x62\xea\x83\x95\xd5\xda\xc3\x2f\x6f\xdf\xfd\x1d\x56\x6b\x84\xca\xbc\x41\xbf\x46\x8b\x61\x0b\x79\xf0\x6b\x63\x5d\x3a\x1a\xc1\x6a\x2d\x1d\x94\x52\x21\x48\x07\x35\xb3\x1e\x4c\x09\xfe\xc2\x5e\xc9\xc2\x32\x7b\xc8\xd2\xd1\xa8\xb9\xf3\xe2\x31\x21\x94\x16\x11\x9c\x29\xfd\x9e\x59\x1c\xc3\xc1\x04\xe0\x4c\x83\x45\x21\x9d\xb7\xb2\x08\x1e\x41\x7a\x60\x5a\x8c\x8c\x85\xad\x11\xb2\x3c\x10\xa4\xf4\x10\xb4\x40\x1b\x5d\x7b\xb4\x5b\xd7\xc5\xf1\xe9\xe6\x0b\xcc\xd1\x39\xb4\xf0\x09\x35\x5a\xa6\xe0\x2e\x14\x4a\x72\x98\x4b\x8e\xda\x21\x30\x07\x35\x7d\x71\x6b\x14\x50\x44\x38\xba\xf8\x91\x42\x59\xb6\xa1\xc0\x47\x13\xb4\x60\x5e\x1a\x3d\x04\x94\x14\x39\xec\xd0\x3a\x69\x34\xfc\xad\x73\xd5\x02\x0e\xc1\x58\x02\xe9\x33\x4f\x09\x58\x30\x35\xdd\x1b\x00\xd3\x07\x50\xcc\x9f\xae\xfe\x00\x21\xa7\xbc\x05\x48\x1d\xdd\xac\x4d\x8d\xe0\xd7\xcc\x53\xd6\x7b\xa9\x14\x14\x08\xc1\x61\x19\xd4\x90\xd0\x8a\xe0\xe1\x7e\xb6\xfa\x7c\xfb\x65\x05\xf9\xcd\x03\xdc\xe7\x8b\x45\x7e\xb3\x7a\xb8\x82\xbd\xf4\x6b\x13\x3c\xe0\x0e\x1b\x28\xb9\xad\x95\x44\x01\x7b\x66\x2d\xd3\xfe\x00\xa6\x24\x84\xdf\xa6\x8b\xc9\xe7\xfc\x66\x95\xbf\x9f\xcd\x67\xab\x07\x30\x16\x3e\xce\x56\x37\xd3\xe5\x12\x3e\xde\x2e\x20\x87\xbb\x7c\xb1\x9a\x4d\xbe\xcc\xf3\x05\xdc\x7d\x59\xdc\xdd\x2e\xa7\x19\x2c\x91\xa2\x42\xba\xff\x7d\xce\xcb\x58\x3d\x8b\x20\xd0\x33\xa9\x5c\xc7\xc4\x83\x09\xe0\xd6\x26\x28\x01\x6b\xb6\x43\xb0\xc8\x51\xee\x50\x00\x03\x6e\xea\xc3\x0f\x17\x95\xb0\x98\x32\xba\x8a\x39\xbf\xda\x90\x30\x2b\x41\x1b\x3f\x04\x87\x08\xff\x58\x7b\x5f\x8f\x47\xa3\xfd\x7e\x9f\x55\x3a\x64\xc6\x56\x23\xd5\xc0\xb9\xd1\x3f\xb3\x94\x30\x6b\x8b\xce\x33\x8f\x2b\xcb\x38\x5a\x30\xc1\xd7\xc1\x3b\x70\xa1\x2c\x25\x97\xa8\x3d\x48\x5d\x1a\xbb\x8d\x9d\x02\xde\x00\xb7\xc8\x3c\x02\x03\x65\x38\x53\x80\x8f\xc8\x43\x3c\x6b\x98\x8e\xed\x6a\x99\x76\x8c\xc7\xaf\xa5\x35\x5b\xca\x35\x38\x4f\x7f\x38\x87\xdb\x42\xa1\x80\x0a\x35\x3a\xe9\xa0\x50\x86\x6f\xb2\xf4\x6b\x9a\x3c\x0b\x86\xfa\x24\x66\xd8\x1a\xc5\xde\xd8\x63\xcf\x22\x14\x41\x2a\x21\x75\x95\xa5\x49\x67\x3d\x06\x1d\x94\x1a\xa6\x11\x42\x19\xb3\x09\x75\xce\xb9\x09\x31\xf6\x3f\x91\xfb\x06\xcc\xd5\xc8\x65\x49\xcd\xc1\x8e\xa7\xde\xc4\xa3\xa3\x5f\x53\x90\x7d\x96\x26\x67\x30\x63\x28\x83\x8e\xe9\xf4\x99\x10\x76\x08\xa2\x18\x7c\x4d\x93\x64\xc7\x2c\x61\xc1\x35\x78\xf3\x19\x1f\xe3\xe1\xe0\x2a\x4d\x12\x59\x42\xdf\xaf\xa5\xcb\x3a\xe0\xdf\x19\xe7\x7f\xc0\xf5\xf5\x75\x1c\xea\x52\x6a\x14\x03\x20\x88\xe4\x25\xb3\xe6\x24\x29\x98\x62\x9a\xe3\x18\x7a\x6f\x1f\x7b\xf0\x33\x88\x22\xab\xd0\xbf\x6f\xbe\x36\xce\x32\x6f\x96\xde\x4a\x5d\xf5\xdf\xfd\x3a\x18\xc6\x5b\xda\xc4\x3b\xd0\x9a\xdf\x98\xa3\x71\x73\xce\x8d\x88\xc7\x6d\xcc\x8d\xd5\xc4\x88\xd6\xa8\xb5\x72\xde\x58\x56\xe1\x18\xbe\x3e\xd1\xef\x27\xca\xea\x29\x4d\x9e\xce\x58\x5e\x36\x46\xaf\xb0\xdc\x42\x00\x6a\x6f\x8f\x7d\x5e\x49\x9a\xd4\xe7\x05\x88\x78\xdf\x2a\xc2\xb2\x0b\xe5\xa2\x08\x1b\x3c\x7c\xbf\x12\x74\x20\xc5\xe3\xf1\x60\x83\x87\xc1\x55\xfa\x6a\x89\xb2\x36\xe8\xdf\xa5\x78\xfc\xd1\x7a\x5d\xdc\x39\xe3\x75\x49\x56\xa7\x78\x07\x83\x0b\x1e\x2d\xba\xa0\x3c\xb5\xbb\xd4\x3b\xb3\x21\xe1\x5a\x13\x3f\x4a\x45\x4a\x4c\x4d\xd5\x72\x8d\x72\x14\x88\x1a\xa4\x47\xcb\x48\x3a\xcd\x0e\x2d\xbd\x1a\x60\xd1\x07\xab\xdd\x91\xc6\x52\x6a\xa6\x3a\xe0\x96\x75\x6f\x19\x6f\x66\xa6\xf9\xfe\x8c\x4b\xee\x1f\x23\x8b\x31\xbb\xd1\x08\x72\x0f\x94\x22\xd4\x46\x6a\x3f\x84\x3d\x82\x46\x14\x34\xf8\x02\x45\xe0\x3e\xe2\xf5\x76\x4c\x05\xec\x35\xc3\x4d\x12\x19\xaf\x9a\x40\x2f\xc1\xb3\xe1\x1f\xc6\x00\xb7\x66\x17\x9f\xb8\x82\xf1\x0d\xb4\x03\x67\xac\xac\xa4\x4e\x5b\x3a\xcf\x86\x8d\x22\xca\x08\x38\x86\x15\x6b\x45\x45\xa4\x2f\xef\x99\x82\x6b\x28\x64\x35\xd3\xfe\xa2\x78\x0d\xe9\xdd\xd5\xc1\x1f\x59\x3b\x3c\x99\x23\xc1\xeb\xff\x32\x18\xc2\xbb\x5f\x8f\x1d\xe1\x0d\x41\xc1\xf7\xc1\xbc\x79\x1d\x2a\xbd\x6c\x86\x97\xaf\x45\x37\x34\xc1\x3f\x47\xaf\x99\x0b\x05\x95\xa3\xc9\x33\xf2\x78\x3e\xc5\x57\xdf\xc0\x3d\xcf\xad\xc3\x6d\xa9\xc9\x98\x10\xaf\x83\x36\x25\xfa\x80\xdc\xe2\x96\x54\x9d\xaa\xc0\x99\x52\x68\x7b\x0e\xa2\x66\x0c\xdb\x76\x8a\xf5\xc2\x6d\xed\x0f\x9d\xd6\x7b\x66\x2b\xf4\xee\xfb\x81\x45\x9c\x37\x6f\x3a\x09\x8c\x54\x1c\x6a\x84\xeb\x6b\xe8\x4d\x16\xd3\x7c\x35\xed\xb5\x63\x34\x1a\xc1\x3d\xc6\x4d\xa8\x50\xb2\x10\xea\x00\x02\x15\x7a\x6c\xe2\x32\x3a\x52\x74\x94\x84\x21\xad\x34\xb4\x6c\xe0\xa3\x74\x5e\xea\x0a\x1a\xa5\xd8\xd3\xbb\xda\xc2\xc5\x19\xe1\x2c\x38\xea\xd6\x8b\x47\xc8\x1b\xda\x28\x2c\x92\xae\x90\xfe\xc7\x71\x63\x4a\x1e\x37\x90\x52\x5a\xe7\xa1\x56\x8c\x63\x46\x78\xc7\x60\x5e\xaf\x6f\x3b\xc9\xe4\x7a\x11\x47\x30\x02\x9d\x1e\x38\xa6\xe8\x81\x24\xf7\x0e\xfa\x1d\xc6\x20\x4d\x12\xdb\x59\x3f\xc3\xbe\x3a\x49\x82\xf3\x58\x3f\x17\x04\x5a\x2c\x70\x87\x24\xa1\x51\x0d\x9a\xc7\x90\x7c\xfd\xeb\xb7\xf6\xf5\x45\x97\xa5\x09\xdd\x7b\x36\xd7\xca\x54\xe7\x73\x2d\x1a\x5a\x78\xb0\x96\xea\x7f\x94\xe0\x92\x66\xfc\xcf\xe0\x3c\x71\x6a\x89\x9e\x56\x2d\x5e\x12\xc9\x28\x89\xf4\xda\x0e\xfe\x2a\x86\xf4\x6e\xc5\x77\x82\xdc\xb5\xaf\x54\xb3\xcd\xd5\xc6\xa3\xf6\x92\x29\x75\xa0\x3a\xec\x2d\xad\x31\xb4\xb8\x0c\xc1\x49\xb2\x8a\x8a\x13\x4d\xa5\xe6\x2a\x88\xa6\x0d\x62\x1f\xb7\x78\x2e\xc6\x7c\xbe\xff\x6c\xd1\x39\x56\x61\x46\x9d\x54\xca\xc7\x76\x83\xd4\xd0\x6b\x44\xae\x3f\xe8\x65\xc7\x20\xcf\x25\x46\x99\x2a\xeb\x9a\x8c\x64\x3a\x17\xc2\xa2\x73\xfd\x41\xab\x39\xc7\xca\xde\xaf\x51\x13\xf9\xa0\x71\x0f\xc7\xd5\x84\x71\x4e\xab\x9a\x18\x02\x13\x82\xa4\xed\x62\x8d\x48\x93\xc4\xed\xa5\xe7\x6b\x88\x9e\x4c\x7d\x9a\xc5\x41\xdb\xff\x9c\x39\x84\x9f\xa6\xff\x5e\x4d\x6e\x3f\x4c\x27\xb7\x77\x0f\x3f\x8d\xe1\xec\xdb\x72\xf6\x9f\xe9\xf1\xdb\xfb\x7c\x9e\xdf\x4c\xa6\x3f\x8d\xe3\xdb\xfc\x42\x42\xde\x74\x29\x90\x43\xe7\x19\xdf\x64\x35\xe2\xa6\xff\xf6\x5c\x07\x4e\x09\x26\x49\x61\x91\x6d\xae\x4e\xc1\x34\x03\xda\xfa\xe8\x24\x17\xae\xe1\x55\xb2\xae\x5e\x8f\x66\xd2\xda\xf7\x3b\x21\x3f\xad\x22\x51\x2a\xbe\x19\x47\x3e\x9f\x1f\x33\xa7\x1f\x44\xc7\xf1\xc3\x87\xe9\x7c\xfa\x29\x5f\x4d\xcf\xac\x96\xab\x7c\x35\x9b\x34\x9f\xfe\x67\x8a\xde\xfd\x30\x45\xbd\xe5\x72\x75\xbb\x98\xf6\xc6\xed\xaf\xf9\x6d\xfe\xa1\xf7\x17\x87\xed\xbe\xf2\xad\x26\xf3\xe6\xde\x58\xf1\xff\xd4\xea\xd9\xee\x50\xb2\x97\x56\x87\x28\x42\xdc\x87\x8b\xd5\x1c\x98\xee\xf4\xa3\x6c\xfe\x3d\x49\xe2\xfd\x17\x15\xe3\x29\x7d\x4a\xff\x1b\x00\x00\xff\xff\x7c\xdb\x3f\x79\x34\x0f\x00\x00") func prestate_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -213,7 +213,7 @@ func prestate_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "prestate_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd0, 0xd5, 0x5, 0x92, 0xed, 0xf4, 0x69, 0x2e, 0x14, 0x48, 0x35, 0x67, 0xcc, 0xf2, 0x3e, 0xc7, 0xf, 0x18, 0x22, 0x7a, 0x4d, 0x6f, 0x31, 0xad, 0x3c, 0x92, 0x77, 0xb4, 0x1, 0x2a, 0xd3, 0x7c}} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xd, 0xb0, 0x72, 0x28, 0xc7, 0x27, 0x97, 0x4d, 0xe, 0xbf, 0x29, 0xe1, 0xa8, 0xd7, 0x52, 0x13, 0xa1, 0x19, 0xc3, 0xfb, 0x8d, 0x5b, 0xcb, 0xdd, 0xa5, 0xd7, 0x98, 0x34, 0x6a, 0xbf, 0x33, 0x6c}} return a, nil } diff --git a/eth/tracers/internal/tracers/prestate_tracer.js b/eth/tracers/internal/tracers/prestate_tracer.js index 99f71d2c33..56aa2b210f 100644 --- a/eth/tracers/internal/tracers/prestate_tracer.js +++ b/eth/tracers/internal/tracers/prestate_tracer.js @@ -40,10 +40,7 @@ var idx = toHex(key); if (this.prestate[acc].storage[idx] === undefined) { - var val = toHex(db.getState(addr, key)); - if (val != "0x0000000000000000000000000000000000000000000000000000000000000000") { - this.prestate[acc].storage[idx] = toHex(db.getState(addr, key)); - } + this.prestate[acc].storage[idx] = toHex(db.getState(addr, key)); } }, From 6a1a4375c680ba4c9ee49ae7cf1e5ab0ab72bdfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 11:36:36 +0200 Subject: [PATCH 36/68] params: set mainnet and Rinkeby Constantinople fork blocks --- params/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/params/config.go b/params/config.go index ba9fac1c4a..7734c6e9cc 100644 --- a/params/config.go +++ b/params/config.go @@ -42,7 +42,7 @@ var ( EIP155Block: big.NewInt(2675000), EIP158Block: big.NewInt(2675000), ByzantiumBlock: big.NewInt(4370000), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(7080000), Ethash: new(EthashConfig), } @@ -90,7 +90,7 @@ var ( EIP155Block: big.NewInt(3), EIP158Block: big.NewInt(3), ByzantiumBlock: big.NewInt(1035301), - ConstantinopleBlock: nil, + ConstantinopleBlock: big.NewInt(3660663), Clique: &CliqueConfig{ Period: 15, Epoch: 30000, From 42a914a84ffcc5d123af400c2cbcd34b05e2b0b3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 15 Oct 2018 12:28:44 +0200 Subject: [PATCH 37/68] cmd/evm, core/vm, eth: implement api methods to do stdjson dump to local filesystem --- cmd/evm/runner.go | 2 +- cmd/evm/staterunner.go | 2 +- .../json_logger.go => core/vm/logger_json.go | 13 +- eth/api_tracer.go | 141 ++++++++++++++++-- internal/web3ext/web3ext.go | 10 ++ 5 files changed, 148 insertions(+), 20 deletions(-) rename cmd/evm/json_logger.go => core/vm/logger_json.go (81%) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index a9c8a38cae..54b67ce101 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -89,7 +89,7 @@ func runCmd(ctx *cli.Context) error { genesisConfig *core.Genesis ) if ctx.GlobalBool(MachineFlag.Name) { - tracer = NewJSONLogger(logconfig, os.Stdout) + tracer = vm.NewJSONLogger(logconfig, os.Stdout) } else if ctx.GlobalBool(DebugFlag.Name) { debugLogger = vm.NewStructLogger(logconfig) tracer = debugLogger diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 06c9be380a..b3c69d9b9d 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -68,7 +68,7 @@ func stateTestCmd(ctx *cli.Context) error { ) switch { case ctx.GlobalBool(MachineFlag.Name): - tracer = NewJSONLogger(config, os.Stderr) + tracer = vm.NewJSONLogger(config, os.Stderr) case ctx.GlobalBool(DebugFlag.Name): debugger = vm.NewStructLogger(config) diff --git a/cmd/evm/json_logger.go b/core/vm/logger_json.go similarity index 81% rename from cmd/evm/json_logger.go rename to core/vm/logger_json.go index 50cb4f0e42..ac3c407597 100644 --- a/cmd/evm/json_logger.go +++ b/core/vm/logger_json.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -package main +package vm import ( "encoding/json" @@ -24,17 +24,16 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core/vm" ) type JSONLogger struct { encoder *json.Encoder - cfg *vm.LogConfig + cfg *LogConfig } // NewJSONLogger creates a new EVM tracer that prints execution steps as JSON objects // into the provided stream. -func NewJSONLogger(cfg *vm.LogConfig, writer io.Writer) *JSONLogger { +func NewJSONLogger(cfg *LogConfig, writer io.Writer) *JSONLogger { return &JSONLogger{json.NewEncoder(writer), cfg} } @@ -43,8 +42,8 @@ func (l *JSONLogger) CaptureStart(from common.Address, to common.Address, create } // CaptureState outputs state information on the logger. -func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { - log := vm.StructLog{ +func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { + log := StructLog{ Pc: pc, Op: op, Gas: gas, @@ -65,7 +64,7 @@ func (l *JSONLogger) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cos } // CaptureFault outputs state information on the logger. -func (l *JSONLogger) CaptureFault(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost uint64, memory *vm.Memory, stack *vm.Stack, contract *vm.Contract, depth int, err error) error { +func (l *JSONLogger) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, memory *Memory, stack *Stack, contract *Contract, depth int, err error) error { return nil } diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 2ebbcc5fd9..77d52d9441 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -17,11 +17,13 @@ package eth import ( + "bufio" "bytes" "context" "errors" "fmt" "io/ioutil" + "os" "runtime" "sync" "time" @@ -60,6 +62,13 @@ type TraceConfig struct { Reexec *uint64 } +// StdTraceConfig holds extra parameters to standard-json trace functions. +type StdTraceConfig struct { + *vm.LogConfig + Reexec *uint64 + TxHash *common.Hash +} + // txTraceResult is the result of a single transaction trace. type txTraceResult struct { Result interface{} `json:"result,omitempty"` // Trace results produced by the tracer @@ -391,13 +400,37 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, return api.TraceBlock(ctx, blob, config) } -// TraceBadBlock returns the structured logs created during the execution of a block -// within the blockchain 'badblocks' cache -func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, index int, config *TraceConfig) ([]*txTraceResult, error) { - if blocks := api.eth.blockchain.BadBlocks(); index < len(blocks) { - return api.traceBlock(ctx, blocks[index], config) +// TraceBadBlockByHash returns the structured logs created during the execution of a block +func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, blockHash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { + blocks := api.eth.blockchain.BadBlocks() + for _, block := range blocks { + if block.Hash() == blockHash { + return api.traceBlock(ctx, block, config) + } } - return nil, fmt.Errorf("index out of range") + return nil, fmt.Errorf("hash not found among bad blocks") +} + +// StandardTraceBadBlockToFile dumps the standard-json logs to files on the local filesystem, +// and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { + blocks := api.eth.blockchain.BadBlocks() + for _, block := range blocks { + if block.Hash() == blockHash { + return api.standardTraceBlockToFile(ctx, block, stdConfig) + } + } + return nil, fmt.Errorf("hash not found among bad blocks") +} + +// StandardTraceBlockToFile dumps the standard-json logs to files on the local filesystem, +// and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { + block := api.eth.blockchain.GetBlockByHash(blockHash) + if block == nil { + return nil, fmt.Errorf("block #%x not found", blockHash) + } + return api.standardTraceBlockToFile(ctx, block, stdConfig) } // traceBlock configures a new tracer according to the provided configuration, and @@ -481,6 +514,92 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, return results, nil } +// standardTraceBlockToFile configures a new tracer which uses standard-json output, and +// traces either a full block or an individual transaction. The return value will be one filename +// per transaction traced. +func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, stdConfig *StdTraceConfig) ([]string, error) { + // Create the parent state database + if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { + return nil, err + } + parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + } + var ( + signer = types.MakeSigner(api.config, block.Number()) + done = false + blockPrefix = fmt.Sprintf("block_0x%x", block.Hash().Bytes()[:4]) + usedLogConfig = &vm.LogConfig{Debug: true} + files []string + reExec_val = defaultTraceReexec + txHash *common.Hash + ) + if stdConfig != nil { + if stdConfig.Reexec != nil { + reExec_val = *stdConfig.Reexec + } + if stdConfig.LogConfig != nil { + usedLogConfig.DisableMemory = stdConfig.LogConfig.DisableMemory + usedLogConfig.DisableStack = stdConfig.LogConfig.DisableStack + usedLogConfig.DisableStorage = stdConfig.LogConfig.DisableStorage + usedLogConfig.Limit = stdConfig.LogConfig.Limit + } + txHash = stdConfig.TxHash + } + statedb, err := api.computeStateDB(parent, reExec_val) + if err != nil { + return nil, err + } + + for i, tx := range block.Transactions() { + var ( + outfile *os.File + err error + ) + msg, _ := tx.AsMessage(signer) + vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) + vmConf := vm.Config{} + if txHash == nil || bytes.Equal(txHash.Bytes(), tx.Hash().Bytes()) { + prefix := fmt.Sprintf("%v-%d-0x%x-", blockPrefix, i, tx.Hash().Bytes()[:4]) + // Open a file to dump trace into + outfile, err = ioutil.TempFile(os.TempDir(), prefix) + if err != nil { + return nil, err + } + files = append(files, outfile.Name()) + vmConf = vm.Config{ + Debug: true, + Tracer: vm.NewJSONLogger(usedLogConfig, bufio.NewWriter(outfile)), + EnablePreimageRecording: true, + } + if txHash != nil { // Only one tx to trace + done = true + } + } + vmenv := vm.NewEVM(vmctx, statedb, api.config, vmConf) + _, _, _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) + + if outfile != nil { + outfile.Close() + log.Info("Wrote trace", "file", outfile.Name()) + } + if err != nil { + return files, err + } + // Finalize the state so any modifications are written to the trie + statedb.Finalise(true) + + if done { + break + } + } + if txHash != nil && !done { + return nil, fmt.Errorf("transaction hash not found in block") + } + return files, nil +} + // computeStateDB retrieves the state database associated with a certain block. // If no state is locally available for the given block, a number of blocks are // attempted to be reexecuted to generate the desired state. @@ -506,7 +625,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* if err != nil { switch err.(type) { case *trie.MissingNodeError: - return nil, errors.New("required historical state unavailable") + return nil, fmt.Errorf("required historical state unavailable (reexec=%d)", reexec) default: return nil, err } @@ -520,7 +639,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* for block.NumberU64() < origin { // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { - log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "elapsed", time.Since(start)) + log.Info("Regenerating historical state", "block", block.NumberU64()+1, "target", origin, "remaining", origin-block.NumberU64()-1, "elapsed", time.Since(start)) logged = time.Now() } // Retrieve the next block to regenerate and process it @@ -529,15 +648,15 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (* } _, _, _, err := api.eth.blockchain.Processor().Process(block, statedb, vm.Config{}) if err != nil { - return nil, err + return nil, fmt.Errorf("processing block %d failed: %v", block.NumberU64(), err) } // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(true) + root, err := statedb.Commit(api.eth.blockchain.Config().IsEIP158(block.Number())) if err != nil { return nil, err } if err := statedb.Reset(root); err != nil { - return nil, err + return nil, fmt.Errorf("state reset after block %d failed: %v", block.NumberU64(), err) } database.TrieDB().Reference(root, common.Hash{}) if proot != (common.Hash{}) { diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index a5f3196535..f36835d8ac 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -384,6 +384,16 @@ web3._extend({ params: 1, inputFormatter: [null] }), + new web3._extend.Method({ + name: 'standardTraceBadBlockToFile', + call: 'debug_standardTraceBadBlockToFile', + params: 2, + }), + new web3._extend.Method({ + name: 'standardTraceBlockToFile', + call: 'debug_standardTraceBlockToFile', + params: 2, + }), new web3._extend.Method({ name: 'traceBlockByNumber', call: 'debug_traceBlockByNumber', From 0983d02aa906394fd2be83ebda104730e32c593b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 12:32:40 +0200 Subject: [PATCH 38/68] eth, internal/web3ext: tiny polishes in tracers --- eth/api_tracer.go | 172 ++++++++++++++++++++---------------- internal/web3ext/web3ext.go | 2 + 2 files changed, 97 insertions(+), 77 deletions(-) diff --git a/eth/api_tracer.go b/eth/api_tracer.go index 77d52d9441..0b8f8aa00b 100644 --- a/eth/api_tracer.go +++ b/eth/api_tracer.go @@ -66,7 +66,7 @@ type TraceConfig struct { type StdTraceConfig struct { *vm.LogConfig Reexec *uint64 - TxHash *common.Hash + TxHash common.Hash } // txTraceResult is the result of a single transaction trace. @@ -375,7 +375,7 @@ func (api *PrivateDebugAPI) TraceBlockByNumber(ctx context.Context, number rpc.B func (api *PrivateDebugAPI) TraceBlockByHash(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { block := api.eth.blockchain.GetBlockByHash(hash) if block == nil { - return nil, fmt.Errorf("block #%x not found", hash) + return nil, fmt.Errorf("block %#x not found", hash) } return api.traceBlock(ctx, block, config) } @@ -400,37 +400,41 @@ func (api *PrivateDebugAPI) TraceBlockFromFile(ctx context.Context, file string, return api.TraceBlock(ctx, blob, config) } -// TraceBadBlockByHash returns the structured logs created during the execution of a block -func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, blockHash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { +// TraceBadBlockByHash returns the structured logs created during the execution of +// EVM against a block pulled from the pool of bad ones and returns them as a JSON +// object. +func (api *PrivateDebugAPI) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { blocks := api.eth.blockchain.BadBlocks() for _, block := range blocks { - if block.Hash() == blockHash { + if block.Hash() == hash { return api.traceBlock(ctx, block, config) } } - return nil, fmt.Errorf("hash not found among bad blocks") + return nil, fmt.Errorf("bad block %#x not found", hash) } -// StandardTraceBadBlockToFile dumps the standard-json logs to files on the local filesystem, -// and returns a list of files to the caller. -func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { +// StandardTraceBlockToFile dumps the structured logs created during the +// execution of EVM to the local file system and returns a list of files +// to the caller. +func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { + block := api.eth.blockchain.GetBlockByHash(hash) + if block == nil { + return nil, fmt.Errorf("block %#x not found", hash) + } + return api.standardTraceBlockToFile(ctx, block, config) +} + +// StandardTraceBadBlockToFile dumps the structured logs created during the +// execution of EVM against a block pulled from the pool of bad ones to the +// local file system and returns a list of files to the caller. +func (api *PrivateDebugAPI) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { blocks := api.eth.blockchain.BadBlocks() for _, block := range blocks { - if block.Hash() == blockHash { - return api.standardTraceBlockToFile(ctx, block, stdConfig) + if block.Hash() == hash { + return api.standardTraceBlockToFile(ctx, block, config) } } - return nil, fmt.Errorf("hash not found among bad blocks") -} - -// StandardTraceBlockToFile dumps the standard-json logs to files on the local filesystem, -// and returns a list of files to the caller. -func (api *PrivateDebugAPI) StandardTraceBlockToFile(ctx context.Context, blockHash common.Hash, stdConfig *StdTraceConfig) ([]string, error) { - block := api.eth.blockchain.GetBlockByHash(blockHash) - if block == nil { - return nil, fmt.Errorf("block #%x not found", blockHash) - } - return api.standardTraceBlockToFile(ctx, block, stdConfig) + return nil, fmt.Errorf("bad block %#x not found", hash) } // traceBlock configures a new tracer according to the provided configuration, and @@ -443,7 +447,7 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { @@ -514,90 +518,104 @@ func (api *PrivateDebugAPI) traceBlock(ctx context.Context, block *types.Block, return results, nil } -// standardTraceBlockToFile configures a new tracer which uses standard-json output, and -// traces either a full block or an individual transaction. The return value will be one filename -// per transaction traced. -func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, stdConfig *StdTraceConfig) ([]string, error) { +// standardTraceBlockToFile configures a new tracer which uses standard JSON output, +// and traces either a full block or an individual transaction. The return value will +// be one filename per transaction traced. +func (api *PrivateDebugAPI) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { + // If we're tracing a single transaction, make sure it's present + if config != nil && config.TxHash != (common.Hash{}) { + var exists bool + for _, tx := range block.Transactions() { + if exists = (tx.Hash() == config.TxHash); exists { + break + } + } + if !exists { + return nil, fmt.Errorf("transaction %#x not found in block", config.TxHash) + } + } // Create the parent state database if err := api.eth.engine.VerifyHeader(api.eth.blockchain, block.Header(), true); err != nil { return nil, err } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } - var ( - signer = types.MakeSigner(api.config, block.Number()) - done = false - blockPrefix = fmt.Sprintf("block_0x%x", block.Hash().Bytes()[:4]) - usedLogConfig = &vm.LogConfig{Debug: true} - files []string - reExec_val = defaultTraceReexec - txHash *common.Hash - ) - if stdConfig != nil { - if stdConfig.Reexec != nil { - reExec_val = *stdConfig.Reexec - } - if stdConfig.LogConfig != nil { - usedLogConfig.DisableMemory = stdConfig.LogConfig.DisableMemory - usedLogConfig.DisableStack = stdConfig.LogConfig.DisableStack - usedLogConfig.DisableStorage = stdConfig.LogConfig.DisableStorage - usedLogConfig.Limit = stdConfig.LogConfig.Limit - } - txHash = stdConfig.TxHash + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec } - statedb, err := api.computeStateDB(parent, reExec_val) + statedb, err := api.computeStateDB(parent, reexec) if err != nil { return nil, err } + // Retrieve the tracing configurations, or use default values + var ( + logConfig vm.LogConfig + txHash common.Hash + ) + if config != nil { + if config.LogConfig != nil { + logConfig = *config.LogConfig + } + txHash = config.TxHash + } + logConfig.Debug = true + // Execute transaction, either tracing all or just the requested one + var ( + signer = types.MakeSigner(api.config, block.Number()) + dumps []string + ) for i, tx := range block.Transactions() { + // Prepare the trasaction for un-traced execution var ( - outfile *os.File - err error + msg, _ = tx.AsMessage(signer) + vmctx = core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) + + vmConf vm.Config + dump *os.File + err error ) - msg, _ := tx.AsMessage(signer) - vmctx := core.NewEVMContext(msg, block.Header(), api.eth.blockchain, nil) - vmConf := vm.Config{} - if txHash == nil || bytes.Equal(txHash.Bytes(), tx.Hash().Bytes()) { - prefix := fmt.Sprintf("%v-%d-0x%x-", blockPrefix, i, tx.Hash().Bytes()[:4]) - // Open a file to dump trace into - outfile, err = ioutil.TempFile(os.TempDir(), prefix) + // If the transaction needs tracing, swap out the configs + if tx.Hash() == txHash || txHash == (common.Hash{}) { + // Generate a unique temporary file to dump it into + prefix := fmt.Sprintf("block_%#x-%d-%#x-", block.Hash().Bytes()[:4], i, tx.Hash().Bytes()[:4]) + + dump, err = ioutil.TempFile(os.TempDir(), prefix) if err != nil { return nil, err } - files = append(files, outfile.Name()) + dumps = append(dumps, dump.Name()) + + // Swap out the noop logger to the standard tracer vmConf = vm.Config{ Debug: true, - Tracer: vm.NewJSONLogger(usedLogConfig, bufio.NewWriter(outfile)), + Tracer: vm.NewJSONLogger(&logConfig, bufio.NewWriter(dump)), EnablePreimageRecording: true, } - if txHash != nil { // Only one tx to trace - done = true - } } + // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, statedb, api.config, vmConf) _, _, _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) - if outfile != nil { - outfile.Close() - log.Info("Wrote trace", "file", outfile.Name()) + if dump != nil { + dump.Close() + log.Info("Wrote standard trace", "file", dump.Name()) } if err != nil { - return files, err + return dumps, err } // Finalize the state so any modifications are written to the trie statedb.Finalise(true) - if done { + // If we've traced the transaction we were looking for, abort + if tx.Hash() == txHash { break } } - if txHash != nil && !done { - return nil, fmt.Errorf("transaction hash not found in block") - } - return files, nil + return dumps, nil } // computeStateDB retrieves the state database associated with a certain block. @@ -675,7 +693,7 @@ func (api *PrivateDebugAPI) TraceTransaction(ctx context.Context, hash common.Ha // Retrieve the transaction and assemble its EVM context tx, blockHash, _, index := rawdb.ReadTransaction(api.eth.ChainDb(), hash) if tx == nil { - return nil, fmt.Errorf("transaction %x not found", hash) + return nil, fmt.Errorf("transaction %#x not found", hash) } reexec := defaultTraceReexec if config != nil && config.Reexec != nil { @@ -755,11 +773,11 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree // Create the parent state database block := api.eth.blockchain.GetBlockByHash(blockHash) if block == nil { - return nil, vm.Context{}, nil, fmt.Errorf("block %x not found", blockHash) + return nil, vm.Context{}, nil, fmt.Errorf("block %#x not found", blockHash) } parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return nil, vm.Context{}, nil, fmt.Errorf("parent %x not found", block.ParentHash()) + return nil, vm.Context{}, nil, fmt.Errorf("parent %#x not found", block.ParentHash()) } statedb, err := api.computeStateDB(parent, reexec) if err != nil { @@ -778,10 +796,10 @@ func (api *PrivateDebugAPI) computeTxEnv(blockHash common.Hash, txIndex int, ree // Not yet the searched for transaction, execute on top of the current state vmenv := vm.NewEVM(context, statedb, api.config, vm.Config{}) if _, _, _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { - return nil, vm.Context{}, nil, fmt.Errorf("tx %x failed: %v", tx.Hash(), err) + return nil, vm.Context{}, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state statedb.Finalise(true) } - return nil, vm.Context{}, nil, fmt.Errorf("tx index %d out of range for block %x", txIndex, blockHash) + return nil, vm.Context{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, blockHash) } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index f36835d8ac..06bfcef69e 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -388,11 +388,13 @@ web3._extend({ name: 'standardTraceBadBlockToFile', call: 'debug_standardTraceBadBlockToFile', params: 2, + inputFormatter: [null, null] }), new web3._extend.Method({ name: 'standardTraceBlockToFile', call: 'debug_standardTraceBlockToFile', params: 2, + inputFormatter: [null, null] }), new web3._extend.Method({ name: 'traceBlockByNumber', From da6e6e7971b093990b964cd708cceab77898a655 Mon Sep 17 00:00:00 2001 From: "Iskander (Alex) Sharipov" Date: Mon, 10 Dec 2018 14:29:34 +0300 Subject: [PATCH 39/68] light: fix duplicated argument in bytes.Equal call Most probably a copy/paste kind of error. Found with gocritic `dupArg` checker. --- light/trie_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/light/trie_test.go b/light/trie_test.go index 51ce9017a3..5b5fce31d9 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -64,7 +64,7 @@ func diffTries(t1, t2 state.Trie) error { spew.Dump(i2) return fmt.Errorf("tries have different keys %x, %x", i1.Key, i2.Key) } - if !bytes.Equal(i2.Value, i2.Value) { + if !bytes.Equal(i1.Value, i2.Value) { return fmt.Errorf("tries differ at key %x", i1.Key) } } From c1e3fe6b140c6ac7f9d7c792480a10208a4ac7ec Mon Sep 17 00:00:00 2001 From: YOSHIDA Masanori Date: Mon, 10 Dec 2018 21:24:55 +0900 Subject: [PATCH 40/68] ethereum: fix typo in interfaces.go (#18266) * Fix typo in interfaces.go * Update interfaces.go --- interfaces.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interfaces.go b/interfaces.go index 26b0fcbc17..be78344062 100644 --- a/interfaces.go +++ b/interfaces.go @@ -146,7 +146,7 @@ type FilterQuery struct { // {{A}} matches topic A in first position // {{}, {B}} matches any topic in first position, B in second position // {{A}, {B}} matches topic A in first position, B in second position - // {{A, B}}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position + // {{A, B}, {C, D}} matches topic (A OR B) in first position, (C OR D) in second position Topics [][]common.Hash } From 48b70ecff1152f9eec091ff03b803d997a573b19 Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Fri, 2 Nov 2018 13:26:45 -0700 Subject: [PATCH 41/68] cmd, eth: Add support for `--whitelist =,...` flag * Rejects peers that respond with a different hash for any of the passed in block numbers. * Meant for emergency situations when the network forks unexpectedly. --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 33 +++++++++++++++++++++++++++++++++ eth/backend.go | 2 +- eth/config.go | 3 +++ eth/handler.go | 26 +++++++++++++++++++++++++- eth/handler_test.go | 4 ++-- eth/helper_test.go | 2 +- 8 files changed, 67 insertions(+), 5 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 69802a48a7..fc1bf461fb 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -87,6 +87,7 @@ var ( utils.LightServFlag, utils.LightPeersFlag, utils.LightKDFFlag, + utils.WhitelistFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 82f17e0ee8..25a702dd73 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -81,6 +81,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.LightServFlag, utils.LightPeersFlag, utils.LightKDFFlag, + utils.WhitelistFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6a285fcb39..4727e7cdf0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -182,6 +182,10 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + WhitelistFlag = cli.StringFlag{ + Name: "whitelist", + Usage: "Comma separated block number-to-hash mappings to enforce (=)", + } // Dashboard settings DashboardEnabledFlag = cli.BoolFlag{ Name: metrics.DashboardEnabledFlag, @@ -1072,6 +1076,34 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) { } } +func setWhitelist(ctx *cli.Context, cfg *eth.Config) { + if ctx.GlobalIsSet(WhitelistFlag.Name) { + entries := strings.Split(ctx.String(WhitelistFlag.Name), ",") + whitelist := make(map[uint64]common.Hash) + for _, entry := range entries { + split := strings.SplitN(entry, "=", 2) + if len(split) != 2 { + Fatalf("invalid whitelist entry: %s", entry) + } + + bn, err := strconv.ParseUint(split[0], 0, 64) + if err != nil { + Fatalf("Invalid whitelist block number %s: %v", split[0], err) + } + + hash := common.Hash{} + err = hash.UnmarshalText([]byte(split[1])) + if err != nil { + Fatalf("Invalid whitelist hash %s: %v", split[1], err) + } + + whitelist[bn] = hash + } + + cfg.Whitelist = whitelist + } +} + // checkExclusive verifies that only a single instance of the provided flags was // set by the user. Each flag might optionally be followed by a string type to // specialize it further. @@ -1137,6 +1169,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { setGPO(ctx, &cfg.GPO) setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) + setWhitelist(ctx, cfg) if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) diff --git a/eth/backend.go b/eth/backend.go index 4721408425..3f538d5f55 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -173,7 +173,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.txPool = core.NewTxPool(config.TxPool, eth.chainConfig, eth.blockchain) - if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb); err != nil { + if eth.protocolManager, err = NewProtocolManager(eth.chainConfig, config.SyncMode, config.NetworkId, eth.eventMux, eth.txPool, eth.engine, eth.blockchain, chainDb, config.Whitelist); err != nil { return nil, err } diff --git a/eth/config.go b/eth/config.go index 601f4735e3..a5c2095124 100644 --- a/eth/config.go +++ b/eth/config.go @@ -87,6 +87,9 @@ type Config struct { SyncMode downloader.SyncMode NoPruning bool + // Whitelist of required block number -> hash values to accept + Whitelist map[uint64]common.Hash `toml:"-"` + // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests LightPeers int `toml:",omitempty"` // Maximum number of LES client peers diff --git a/eth/handler.go b/eth/handler.go index 741fc9d5a5..21f31bc7c1 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,6 +17,7 @@ package eth import ( + "bytes" "encoding/json" "errors" "fmt" @@ -88,6 +89,8 @@ type ProtocolManager struct { txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription + whitelist map[uint64]common.Hash + // channels for fetcher, syncer, txsyncLoop newPeerCh chan *peer txsyncCh chan *txsync @@ -101,7 +104,7 @@ type ProtocolManager struct { // NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable // with the Ethereum network. -func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkID uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) { +func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, networkID uint64, mux *event.TypeMux, txpool txPool, engine consensus.Engine, blockchain *core.BlockChain, chaindb ethdb.Database, whitelist map[uint64]common.Hash) (*ProtocolManager, error) { // Create the protocol manager with the base fields manager := &ProtocolManager{ networkID: networkID, @@ -110,6 +113,7 @@ func NewProtocolManager(config *params.ChainConfig, mode downloader.SyncMode, ne blockchain: blockchain, chainconfig: config, peers: newPeerSet(), + whitelist: whitelist, newPeerCh: make(chan *peer), noMorePeers: make(chan struct{}), txsyncCh: make(chan *txsync), @@ -307,6 +311,16 @@ func (pm *ProtocolManager) handle(p *peer) error { } }() } + + // If we have any explicit whitelist block hashes, request them + for bn := range pm.whitelist { + p.Log().Debug("Requesting whitelist block", "number", bn) + if err := p.RequestHeadersByNumber(bn, 1, 0, false); err != nil { + p.Log().Error("whitelist request failed", "err", err, "number", bn, "peer", p.id) + return err + } + } + // main loop. handle incoming messages. for { if err := pm.handleMsg(p); err != nil { @@ -452,6 +466,16 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested headers, deliver the rest to the downloader filter := len(headers) == 1 if filter { + // Check for any responses not matching our whitelist + if expected, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { + actual := headers[0].Hash() + if !bytes.Equal(expected.Bytes(), actual.Bytes()) { + p.Log().Info("Dropping peer with non-matching whitelist block", "number", headers[0].Number.Uint64(), "hash", actual, "expected", expected) + return errors.New("whitelist block mismatch") + } + p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", expected) + } + // If it's a potential DAO fork check, validate against the rules if p.forkDrop != nil && pm.chainconfig.DAOForkBlock.Cmp(headers[0].Number) == 0 { // Disable the fork drop timer diff --git a/eth/handler_test.go b/eth/handler_test.go index 44824fd0b7..9fffd95811 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -478,7 +478,7 @@ func testDAOChallenge(t *testing.T, localForked, remoteForked bool, timeout bool if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } @@ -559,7 +559,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db) + pm, err := NewProtocolManager(config, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } diff --git a/eth/helper_test.go b/eth/helper_test.go index 4e38a129e0..b18a02baf9 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -66,7 +66,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func panic(err) } - pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db) + pm, err := NewProtocolManager(gspec.Config, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, nil) if err != nil { return nil, nil, err } From 31b33349227715e8d1f8753d913892c1e35439d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 14:47:01 +0200 Subject: [PATCH 42/68] cmd/utils, eth: minor polishes on whitelist code --- cmd/utils/flags.go | 41 ++++++++++++++++++----------------------- eth/handler.go | 29 +++++++++++------------------ 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4727e7cdf0..c9115d7a35 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1077,30 +1077,25 @@ func setEthash(ctx *cli.Context, cfg *eth.Config) { } func setWhitelist(ctx *cli.Context, cfg *eth.Config) { - if ctx.GlobalIsSet(WhitelistFlag.Name) { - entries := strings.Split(ctx.String(WhitelistFlag.Name), ",") - whitelist := make(map[uint64]common.Hash) - for _, entry := range entries { - split := strings.SplitN(entry, "=", 2) - if len(split) != 2 { - Fatalf("invalid whitelist entry: %s", entry) - } - - bn, err := strconv.ParseUint(split[0], 0, 64) - if err != nil { - Fatalf("Invalid whitelist block number %s: %v", split[0], err) - } - - hash := common.Hash{} - err = hash.UnmarshalText([]byte(split[1])) - if err != nil { - Fatalf("Invalid whitelist hash %s: %v", split[1], err) - } - - whitelist[bn] = hash + whitelist := ctx.GlobalString(WhitelistFlag.Name) + if whitelist == "" { + return + } + cfg.Whitelist = make(map[uint64]common.Hash) + for _, entry := range strings.Split(whitelist, ",") { + parts := strings.Split(entry, "=") + if len(parts) != 2 { + Fatalf("Invalid whitelist entry: %s", entry) } - - cfg.Whitelist = whitelist + number, err := strconv.ParseUint(parts[0], 0, 64) + if err != nil { + Fatalf("Invalid whitelist block number %s: %v", parts[0], err) + } + var hash common.Hash + if err = hash.UnmarshalText([]byte(parts[1])); err != nil { + Fatalf("Invalid whitelist hash %s: %v", parts[1], err) + } + cfg.Whitelist[number] = hash } } diff --git a/eth/handler.go b/eth/handler.go index 21f31bc7c1..b42612a566 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,7 +17,6 @@ package eth import ( - "bytes" "encoding/json" "errors" "fmt" @@ -311,17 +310,13 @@ func (pm *ProtocolManager) handle(p *peer) error { } }() } - // If we have any explicit whitelist block hashes, request them - for bn := range pm.whitelist { - p.Log().Debug("Requesting whitelist block", "number", bn) - if err := p.RequestHeadersByNumber(bn, 1, 0, false); err != nil { - p.Log().Error("whitelist request failed", "err", err, "number", bn, "peer", p.id) + for number := range pm.whitelist { + if err := p.RequestHeadersByNumber(number, 1, 0, false); err != nil { return err } } - - // main loop. handle incoming messages. + // Handle incoming messages until the connection is torn down for { if err := pm.handleMsg(p); err != nil { p.Log().Debug("Ethereum message handling failed", "err", err) @@ -466,16 +461,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested headers, deliver the rest to the downloader filter := len(headers) == 1 if filter { - // Check for any responses not matching our whitelist - if expected, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { - actual := headers[0].Hash() - if !bytes.Equal(expected.Bytes(), actual.Bytes()) { - p.Log().Info("Dropping peer with non-matching whitelist block", "number", headers[0].Number.Uint64(), "hash", actual, "expected", expected) - return errors.New("whitelist block mismatch") - } - p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", expected) - } - // If it's a potential DAO fork check, validate against the rules if p.forkDrop != nil && pm.chainconfig.DAOForkBlock.Cmp(headers[0].Number) == 0 { // Disable the fork drop timer @@ -490,6 +475,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.Log().Debug("Verified to be on the same side of the DAO fork") return nil } + // Otherwise if it's a whitelisted block, validate against the set + if want, ok := pm.whitelist[headers[0].Number.Uint64()]; ok { + if hash := headers[0].Hash(); want != hash { + p.Log().Info("Whitelist mismatch, dropping peer", "number", headers[0].Number.Uint64(), "hash", hash, "want", want) + return errors.New("whitelist block mismatch") + } + p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) + } // Irrelevant of the fork checks, send the header to the fetcher just in case headers = pm.fetcher.FilterHeaders(p.id, headers, time.Now()) } From 362e2ba792927282c41acda05cb1a7cd34e1c790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 10 Dec 2018 14:28:50 +0200 Subject: [PATCH 43/68] params: update CHTs for the 1.8.20 release --- params/config.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/params/config.go b/params/config.go index 7734c6e9cc..2935ef1f3f 100644 --- a/params/config.go +++ b/params/config.go @@ -49,10 +49,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "mainnet", - SectionIndex: 206, - SectionHead: common.HexToHash("0x9fa677c7c0580136f5a86d9b2fd29b112e531f0284396298b8809bcb6787b538"), - CHTRoot: common.HexToHash("0x7f32dfb29e341b4c8c10ea2e06a812bcea470366f635b7a8b3d0856684cd76f4"), - BloomRoot: common.HexToHash("0x0169e174f0a8172aec217556d8a25c7ba7ca52aacff170325230a75740ff1eaf"), + SectionIndex: 208, + SectionHead: common.HexToHash("0x5e9f7696c397d9df8f3b1abda857753575c6f5cff894e1a3d9e1a2af1bd9d6ac"), + CHTRoot: common.HexToHash("0x954a63134f6897f015f026387c59c98c4dae7b336610ff5a143455aac9153e9d"), + BloomRoot: common.HexToHash("0x8006c5e44b14d90d7cc9cd5fa1cb48cf53697ee3bbbf4b76fdfa70b0242500a9"), } // TestnetChainConfig contains the chain parameters to run a node on the Ropsten test network. @@ -73,10 +73,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ Name: "testnet", - SectionIndex: 136, - SectionHead: common.HexToHash("0xe5d80bb08d92bbc12dfe510c64cba01eafcbb4ba585e7c7ab7f8a93c6f295ab3"), - CHTRoot: common.HexToHash("0xe3ca77ab0cb51eec74f4f7458e36aee207c68768387b39cb0bcff0940a6264d8"), - BloomRoot: common.HexToHash("0x30c8eeadac5539d3dcd6e88915d1a07cb2f3a1d6ebe7e553e3ee783c04c68c2d"), + SectionIndex: 139, + SectionHead: common.HexToHash("0x9fad89a5e3b993c8339b9cf2cbbeb72cd08774ea6b71b105b3dd880420c618f4"), + CHTRoot: common.HexToHash("0xc815833881989c5d2035147e1a79a33d22cbc5313e104ff01e6ab405bd28b317"), + BloomRoot: common.HexToHash("0xd94ee9f3c480858f53ec5d059aebdbb2e8d904702f100875ee59ec5f366e841d"), } // RinkebyChainConfig contains the chain parameters to run a node on the Rinkeby test network. @@ -100,10 +100,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ Name: "rinkeby", - SectionIndex: 103, - SectionHead: common.HexToHash("0x9f38b903852831bf4fa7992f7fd43d8b26da2deb82b421fb845cf6faee54e056"), - CHTRoot: common.HexToHash("0x2d710c2cea468d2e604838000d658ee213e4abb07f90c4f71f5cd7f8510aa708"), - BloomRoot: common.HexToHash("0xcc401060280c2cc82697ea5ecef8cac61e52063c37533a2e9609332419704d5f"), + SectionIndex: 105, + SectionHead: common.HexToHash("0xec8147d43f936258aaf1b9b9ec91b0a853abf7109f436a23649be809ea43d507"), + CHTRoot: common.HexToHash("0xd92703b444846a3db928e87e450770e5d5cbe193131dc8f7c4cf18b4de925a75"), + BloomRoot: common.HexToHash("0xff45a6f807138a2cde0cea0c209d9ce5ad8e43ccaae5a7c41af801bb72a1ef96"), } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced From dd98d1da9464c9ae6e1da69d7a41d4551f20bd8d Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 10 Dec 2018 13:56:01 +0100 Subject: [PATCH 44/68] swarm/network: Correct ambiguity in compared addresses (#18251) --- swarm/network/networkid_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/swarm/network/networkid_test.go b/swarm/network/networkid_test.go index d1d359de68..191d67e5b6 100644 --- a/swarm/network/networkid_test.go +++ b/swarm/network/networkid_test.go @@ -95,8 +95,7 @@ func TestNetworkID(t *testing.T) { kademlias[node].EachAddr(nil, 0, func(addr *BzzAddr, _ int, _ bool) bool { found := false for _, nd := range netIDGroup { - p := nd.Bytes() - if bytes.Equal(p, addr.Address()) { + if bytes.Equal(kademlias[nd].BaseAddr(), addr.Address()) { found = true } } From bb724080cac9fa36ec6b638cfd5cf0e54bc23362 Mon Sep 17 00:00:00 2001 From: Elad Date: Tue, 11 Dec 2018 13:51:58 +0530 Subject: [PATCH 45/68] cmd/swarm, metrics, swarm/api/client, swarm/storage, swarm/metrics, swarm/api/http: add instrumentation (#18274) --- cmd/swarm/swarm-smoke/feed_upload_and_sync.go | 64 ++++++- cmd/swarm/swarm-smoke/main.go | 71 ++++++++ cmd/swarm/swarm-smoke/upload_and_sync.go | 160 ++++++++++++------ metrics/influxdb/influxdb.go | 28 +++ swarm/api/client/client.go | 76 ++++++++- swarm/api/http/middleware.go | 11 +- swarm/metrics/flags.go | 32 ++-- swarm/storage/chunker.go | 8 + 8 files changed, 368 insertions(+), 82 deletions(-) diff --git a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go index 7ec1528263..2c5e3fd235 100644 --- a/cmd/swarm/swarm-smoke/feed_upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/feed_upload_and_sync.go @@ -2,11 +2,13 @@ package main import ( "bytes" + "context" "crypto/md5" "fmt" "io" "io/ioutil" "net/http" + "net/http/httptrace" "os" "os/exec" "strings" @@ -16,8 +18,13 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/ethereum/go-ethereum/swarm/testutil" colorable "github.com/mattn/go-colorable" + opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" ) @@ -26,11 +33,29 @@ const ( feedRandomDataLength = 8 ) -// TODO: retrieve with manifest + extract repeating code func cliFeedUploadAndSync(c *cli.Context) error { - + metrics.GetOrRegisterCounter("feed-and-sync", nil).Inc(1) log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))) + errc := make(chan error) + go func() { + errc <- feedUploadAndSync(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("feed-and-sync.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("feed-and-sync.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +// TODO: retrieve with manifest + extract repeating code +func feedUploadAndSync(c *cli.Context) error { defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "size (kb)", filesize) }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) @@ -204,12 +229,12 @@ func cliFeedUploadAndSync(c *cli.Context) error { log.Info("all endpoints synced random data successfully") // upload test file - log.Info("uploading to " + endpoints[0] + " and syncing") + seed := int(time.Now().UnixNano() / 1e6) + log.Info("feed uploading to "+endpoints[0]+" and syncing", "seed", seed) - f, cleanup := generateRandomFile(filesize * 1000) - defer cleanup() + randomBytes := testutil.RandomBytes(seed, filesize*1000) - hash, err := upload(f, endpoints[0]) + hash, err := upload(&randomBytes, endpoints[0]) if err != nil { return err } @@ -218,7 +243,7 @@ func cliFeedUploadAndSync(c *cli.Context) error { return err } multihashHex := hexutil.Encode(hashBytes) - fileHash, err := digest(f) + fileHash, err := digest(bytes.NewReader(randomBytes)) if err != nil { return err } @@ -284,14 +309,37 @@ func cliFeedUploadAndSync(c *cli.Context) error { } func fetchFeed(topic string, user string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "feed-and-sync.fetch") + defer sp.Finish() + log.Trace("sleeping", "ruid", ruid) time.Sleep(3 * time.Second) log.Trace("http get request (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user) - res, err := http.Get(endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user) + + var tn time.Time + reqUri := endpoint + "/bzz-feed:/?topic=" + topic + "&user=" + user + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("feed-and-sync - http get", "feed-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { + log.Error(err.Error(), "ruid", ruid) return err } + log.Trace("http get response (feed)", "ruid", ruid, "api", endpoint, "topic", topic, "user", user, "code", res.StatusCode, "len", res.ContentLength) if res.StatusCode != 200 { diff --git a/cmd/swarm/swarm-smoke/main.go b/cmd/swarm/swarm-smoke/main.go index 845998dc12..66cecdc5c7 100644 --- a/cmd/swarm/swarm-smoke/main.go +++ b/cmd/swarm/swarm-smoke/main.go @@ -17,14 +17,25 @@ package main import ( + "fmt" "os" "sort" + "github.com/ethereum/go-ethereum/cmd/utils" + gethmetrics "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/metrics/influxdb" + swarmmetrics "github.com/ethereum/go-ethereum/swarm/metrics" + "github.com/ethereum/go-ethereum/swarm/tracing" + "github.com/ethereum/go-ethereum/log" cli "gopkg.in/urfave/cli.v1" ) +var ( + gitCommit string // Git SHA1 commit hash of the release (set via linker flags) +) + var ( endpoints []string includeLocalhost bool @@ -32,9 +43,12 @@ var ( appName string scheme string filesize int + syncDelay int from int to int verbosity int + timeout int + single bool ) func main() { @@ -85,14 +99,42 @@ func main() { Usage: "file size for generated random file in KB", Destination: &filesize, }, + cli.IntFlag{ + Name: "sync-delay", + Value: 5, + Usage: "duration of delay in seconds to wait for content to be synced", + Destination: &syncDelay, + }, cli.IntFlag{ Name: "verbosity", Value: 1, Usage: "verbosity", Destination: &verbosity, }, + cli.IntFlag{ + Name: "timeout", + Value: 120, + Usage: "timeout in seconds after which kill the process", + Destination: &timeout, + }, + cli.BoolFlag{ + Name: "single", + Usage: "whether to fetch content from a single node or from all nodes", + Destination: &single, + }, } + app.Flags = append(app.Flags, []cli.Flag{ + utils.MetricsEnabledFlag, + swarmmetrics.MetricsInfluxDBEndpointFlag, + swarmmetrics.MetricsInfluxDBDatabaseFlag, + swarmmetrics.MetricsInfluxDBUsernameFlag, + swarmmetrics.MetricsInfluxDBPasswordFlag, + swarmmetrics.MetricsInfluxDBHostTagFlag, + }...) + + app.Flags = append(app.Flags, tracing.Flags...) + app.Commands = []cli.Command{ { Name: "upload_and_sync", @@ -111,9 +153,38 @@ func main() { sort.Sort(cli.FlagsByName(app.Flags)) sort.Sort(cli.CommandsByName(app.Commands)) + app.Before = func(ctx *cli.Context) error { + tracing.Setup(ctx) + return nil + } + + app.After = func(ctx *cli.Context) error { + return emitMetrics(ctx) + } + err := app.Run(os.Args) if err != nil { log.Error(err.Error()) + os.Exit(1) } } + +func emitMetrics(ctx *cli.Context) error { + if gethmetrics.Enabled { + var ( + endpoint = ctx.GlobalString(swarmmetrics.MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(swarmmetrics.MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(swarmmetrics.MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(swarmmetrics.MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(swarmmetrics.MetricsInfluxDBHostTagFlag.Name) + ) + return influxdb.InfluxDBWithTagsOnce(gethmetrics.DefaultRegistry, endpoint, database, username, password, "swarm-smoke.", map[string]string{ + "host": hosttag, + "version": gitCommit, + "filesize": fmt.Sprintf("%v", filesize), + }) + } + + return nil +} diff --git a/cmd/swarm/swarm-smoke/upload_and_sync.go b/cmd/swarm/swarm-smoke/upload_and_sync.go index 3843457dc4..d605f79a3a 100644 --- a/cmd/swarm/swarm-smoke/upload_and_sync.go +++ b/cmd/swarm/swarm-smoke/upload_and_sync.go @@ -18,21 +18,27 @@ package main import ( "bytes" + "context" "crypto/md5" crand "crypto/rand" - "crypto/tls" "errors" "fmt" "io" "io/ioutil" + "math/rand" "net/http" + "net/http/httptrace" "os" - "os/exec" - "strings" "sync" "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/swarm/spancontext" + "github.com/ethereum/go-ethereum/swarm/testutil" + opentracing "github.com/opentracing/opentracing-go" "github.com/pborman/uuid" cli "gopkg.in/urfave/cli.v1" @@ -40,11 +46,11 @@ import ( func generateEndpoints(scheme string, cluster string, app string, from int, to int) { if cluster == "prod" { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%v.swarm-gateways.net", scheme, port)) } } else { - for port := from; port <= to; port++ { + for port := from; port < to; port++ { endpoints = append(endpoints, fmt.Sprintf("%s://%s-%v-%s.stg.swarm-gateways.net", scheme, app, port, cluster)) } } @@ -58,22 +64,48 @@ func cliUploadAndSync(c *cli.Context) error { log.PrintOrigins(true) log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(verbosity), log.StreamHandler(os.Stdout, log.TerminalFormat(true)))) - defer func(now time.Time) { log.Info("total time", "time", time.Since(now), "kb", filesize) }(time.Now()) + metrics.GetOrRegisterCounter("upload-and-sync", nil).Inc(1) + + errc := make(chan error) + go func() { + errc <- uploadAndSync(c) + }() + + select { + case err := <-errc: + if err != nil { + metrics.GetOrRegisterCounter("upload-and-sync.fail", nil).Inc(1) + } + return err + case <-time.After(time.Duration(timeout) * time.Second): + metrics.GetOrRegisterCounter("upload-and-sync.timeout", nil).Inc(1) + return fmt.Errorf("timeout after %v sec", timeout) + } +} + +func uploadAndSync(c *cli.Context) error { + defer func(now time.Time) { + totalTime := time.Since(now) + + log.Info("total time", "time", totalTime, "kb", filesize) + metrics.GetOrRegisterCounter("upload-and-sync.total-time", nil).Inc(int64(totalTime)) + }(time.Now()) generateEndpoints(scheme, cluster, appName, from, to) + seed := int(time.Now().UnixNano() / 1e6) + log.Info("uploading to "+endpoints[0]+" and syncing", "seed", seed) - log.Info("uploading to " + endpoints[0] + " and syncing") + randomBytes := testutil.RandomBytes(seed, filesize*1000) - f, cleanup := generateRandomFile(filesize * 1000) - defer cleanup() - - hash, err := upload(f, endpoints[0]) + t1 := time.Now() + hash, err := upload(&randomBytes, endpoints[0]) if err != nil { log.Error(err.Error()) return err } + metrics.GetOrRegisterCounter("upload-and-sync.upload-time", nil).Inc(int64(time.Since(t1))) - fhash, err := digest(f) + fhash, err := digest(bytes.NewReader(randomBytes)) if err != nil { log.Error(err.Error()) return err @@ -81,23 +113,47 @@ func cliUploadAndSync(c *cli.Context) error { log.Info("uploaded successfully", "hash", hash, "digest", fmt.Sprintf("%x", fhash)) - time.Sleep(3 * time.Second) + time.Sleep(time.Duration(syncDelay) * time.Second) wg := sync.WaitGroup{} - for _, endpoint := range endpoints { + if single { + rand.Seed(time.Now().UTC().UnixNano()) + randIndex := 1 + rand.Intn(len(endpoints)-1) ruid := uuid.New()[:8] wg.Add(1) go func(endpoint string, ruid string) { for { + start := time.Now() err := fetch(hash, endpoint, fhash, ruid) + fetchTime := time.Since(start) if err != nil { continue } + metrics.GetOrRegisterMeter("upload-and-sync.single.fetch-time", nil).Mark(int64(fetchTime)) wg.Done() return } - }(endpoint, ruid) + }(endpoints[randIndex], ruid) + } else { + for _, endpoint := range endpoints { + ruid := uuid.New()[:8] + wg.Add(1) + go func(endpoint string, ruid string) { + for { + start := time.Now() + err := fetch(hash, endpoint, fhash, ruid) + fetchTime := time.Since(start) + if err != nil { + continue + } + + metrics.GetOrRegisterMeter("upload-and-sync.each.fetch-time", nil).Mark(int64(fetchTime)) + wg.Done() + return + } + }(endpoint, ruid) + } } wg.Wait() log.Info("all endpoints synced random file successfully") @@ -107,16 +163,33 @@ func cliUploadAndSync(c *cli.Context) error { // fetch is getting the requested `hash` from the `endpoint` and compares it with the `original` file func fetch(hash string, endpoint string, original []byte, ruid string) error { + ctx, sp := spancontext.StartSpan(context.Background(), "upload-and-sync.fetch") + defer sp.Finish() + log.Trace("sleeping", "ruid", ruid) time.Sleep(3 * time.Second) - log.Trace("http get request", "ruid", ruid, "api", endpoint, "hash", hash) - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - }} - res, err := client.Get(endpoint + "/bzz:/" + hash + "/") + + var tn time.Time + reqUri := endpoint + "/bzz:/" + hash + "/" + req, _ := http.NewRequest("GET", reqUri, nil) + + opentracing.GlobalTracer().Inject( + sp.Context(), + opentracing.HTTPHeaders, + opentracing.HTTPHeadersCarrier(req.Header)) + + trace := client.GetClientTrace("upload-and-sync - http get", "upload-and-sync", ruid, &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + + //transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { - log.Warn(err.Error(), "ruid", ruid) + log.Error(err.Error(), "ruid", ruid) return err } log.Trace("http get response", "ruid", ruid, "api", endpoint, "hash", hash, "code", res.StatusCode, "len", res.ContentLength) @@ -147,16 +220,19 @@ func fetch(hash string, endpoint string, original []byte, ruid string) error { } // upload is uploading a file `f` to `endpoint` via the `swarm up` cmd -func upload(f *os.File, endpoint string) (string, error) { - var out bytes.Buffer - cmd := exec.Command("swarm", "--bzzapi", endpoint, "up", f.Name()) - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - return "", err +func upload(dataBytes *[]byte, endpoint string) (string, error) { + swarm := client.NewClient(endpoint) + f := &client.File{ + ReadCloser: ioutil.NopCloser(bytes.NewReader(*dataBytes)), + ManifestEntry: api.ManifestEntry{ + ContentType: "text/plain", + Mode: 0660, + Size: int64(len(*dataBytes)), + }, } - hash := strings.TrimRight(out.String(), "\r\n") - return hash, nil + + // upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded. + return swarm.Upload(f, "", false) } func digest(r io.Reader) ([]byte, error) { @@ -179,27 +255,3 @@ func generateRandomData(datasize int) ([]byte, error) { } return b, nil } - -// generateRandomFile is creating a temporary file with the requested byte size -func generateRandomFile(size int) (f *os.File, teardown func()) { - // create a tmp file - tmp, err := ioutil.TempFile("", "swarm-test") - if err != nil { - panic(err) - } - - // callback for tmp file cleanup - teardown = func() { - tmp.Close() - os.Remove(tmp.Name()) - } - - buf := make([]byte, size) - _, err = crand.Read(buf) - if err != nil { - panic(err) - } - ioutil.WriteFile(tmp.Name(), buf, 0755) - - return tmp, teardown -} diff --git a/metrics/influxdb/influxdb.go b/metrics/influxdb/influxdb.go index 31a5c21b5f..c4ef927234 100644 --- a/metrics/influxdb/influxdb.go +++ b/metrics/influxdb/influxdb.go @@ -58,6 +58,34 @@ func InfluxDBWithTags(r metrics.Registry, d time.Duration, url, database, userna rep.run() } +// InfluxDBWithTagsOnce runs once an InfluxDB reporter and post the given metrics.Registry with the specified tags +func InfluxDBWithTagsOnce(r metrics.Registry, url, database, username, password, namespace string, tags map[string]string) error { + u, err := uurl.Parse(url) + if err != nil { + return fmt.Errorf("Unable to parse InfluxDB. url: %s, err: %v", url, err) + } + + rep := &reporter{ + reg: r, + url: *u, + database: database, + username: username, + password: password, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + if err := rep.makeClient(); err != nil { + return fmt.Errorf("Unable to make InfluxDB client. err: %v", err) + } + + if err := rep.send(); err != nil { + return fmt.Errorf("Unable to send to InfluxDB. err: %v", err) + } + + return nil +} + func (r *reporter) makeClient() (err error) { r.client, err = client.NewClient(client.Config{ URL: r.url, diff --git a/swarm/api/client/client.go b/swarm/api/client/client.go index d9837ca732..f793ca8b89 100644 --- a/swarm/api/client/client.go +++ b/swarm/api/client/client.go @@ -19,6 +19,7 @@ package client import ( "archive/tar" "bytes" + "context" "encoding/json" "errors" "fmt" @@ -26,6 +27,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" + "net/http/httptrace" "net/textproto" "net/url" "os" @@ -33,9 +35,14 @@ import ( "regexp" "strconv" "strings" + "time" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/spancontext" "github.com/ethereum/go-ethereum/swarm/storage/feed" + "github.com/pborman/uuid" ) var ( @@ -474,6 +481,11 @@ type UploadFn func(file *File) error // TarUpload uses the given Uploader to upload files to swarm as a tar stream, // returning the resulting manifest hash func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, toEncrypt bool) (string, error) { + ctx, sp := spancontext.StartSpan(context.Background(), "api.client.tarupload") + defer sp.Finish() + + var tn time.Time + reqR, reqW := io.Pipe() defer reqR.Close() addr := hash @@ -489,6 +501,12 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t if err != nil { return "", err } + + trace := GetClientTrace("swarm api client - upload tar", "api.client.uploadtar", uuid.New()[:8], &tn) + + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + transport := http.DefaultTransport + req.Header.Set("Content-Type", "application/x-tar") if defaultPath != "" { q := req.URL.Query() @@ -529,8 +547,8 @@ func (c *Client) TarUpload(hash string, uploader Uploader, defaultPath string, t } reqW.CloseWithError(err) }() - - res, err := http.DefaultClient.Do(req) + tn = time.Now() + res, err := transport.RoundTrip(req) if err != nil { return "", err } @@ -728,3 +746,57 @@ func (c *Client) GetFeedRequest(query *feed.Query, manifestAddressOrDomain strin } return &metadata, nil } + +func GetClientTrace(traceMsg, metricPrefix, ruid string, tn *time.Time) *httptrace.ClientTrace { + trace := &httptrace.ClientTrace{ + GetConn: func(_ string) { + log.Trace(traceMsg+" - http get", "event", "GetConn", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".getconn", nil).Update(time.Since(*tn)) + }, + GotConn: func(_ httptrace.GotConnInfo) { + log.Trace(traceMsg+" - http get", "event", "GotConn", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".gotconn", nil).Update(time.Since(*tn)) + }, + PutIdleConn: func(err error) { + log.Trace(traceMsg+" - http get", "event", "PutIdleConn", "ruid", ruid, "err", err) + metrics.GetOrRegisterResettingTimer(metricPrefix+".putidle", nil).Update(time.Since(*tn)) + }, + GotFirstResponseByte: func() { + log.Trace(traceMsg+" - http get", "event", "GotFirstResponseByte", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".firstbyte", nil).Update(time.Since(*tn)) + }, + Got100Continue: func() { + log.Trace(traceMsg, "event", "Got100Continue", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".got100continue", nil).Update(time.Since(*tn)) + }, + DNSStart: func(_ httptrace.DNSStartInfo) { + log.Trace(traceMsg, "event", "DNSStart", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsstart", nil).Update(time.Since(*tn)) + }, + DNSDone: func(_ httptrace.DNSDoneInfo) { + log.Trace(traceMsg, "event", "DNSDone", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".dnsdone", nil).Update(time.Since(*tn)) + }, + ConnectStart: func(network, addr string) { + log.Trace(traceMsg, "event", "ConnectStart", "ruid", ruid, "network", network, "addr", addr) + metrics.GetOrRegisterResettingTimer(metricPrefix+".connectstart", nil).Update(time.Since(*tn)) + }, + ConnectDone: func(network, addr string, err error) { + log.Trace(traceMsg, "event", "ConnectDone", "ruid", ruid, "network", network, "addr", addr, "err", err) + metrics.GetOrRegisterResettingTimer(metricPrefix+".connectdone", nil).Update(time.Since(*tn)) + }, + WroteHeaders: func() { + log.Trace(traceMsg, "event", "WroteHeaders(request)", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wroteheaders", nil).Update(time.Since(*tn)) + }, + Wait100Continue: func() { + log.Trace(traceMsg, "event", "Wait100Continue", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wait100continue", nil).Update(time.Since(*tn)) + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { + log.Trace(traceMsg, "event", "WroteRequest", "ruid", ruid) + metrics.GetOrRegisterResettingTimer(metricPrefix+".wroterequest", nil).Update(time.Since(*tn)) + }, + } + return trace +} diff --git a/swarm/api/http/middleware.go b/swarm/api/http/middleware.go index 115a008566..f7f819eab5 100644 --- a/swarm/api/http/middleware.go +++ b/swarm/api/http/middleware.go @@ -74,13 +74,15 @@ func ParseURI(h http.Handler) http.Handler { func InitLoggingResponseWriter(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - startTime := time.Now() - defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(startTime) + tn := time.Now() writer := newLoggingResponseWriter(w) h.ServeHTTP(writer, r) - log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode) - metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).UpdateSince(startTime) + + ts := time.Since(tn) + log.Info("request served", "ruid", GetRUID(r.Context()), "code", writer.statusCode, "time", ts*time.Millisecond) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).Update(ts) + metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.%d.time", r.Method, writer.statusCode), nil).Update(ts) }) } @@ -93,6 +95,7 @@ func InstrumentOpenTracing(h http.Handler) http.Handler { } spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme) ctx, sp := spancontext.StartSpan(r.Context(), spanName) + defer sp.Finish() h.ServeHTTP(w, r.WithContext(ctx)) }) diff --git a/swarm/metrics/flags.go b/swarm/metrics/flags.go index 79490fd360..7c12120a60 100644 --- a/swarm/metrics/flags.go +++ b/swarm/metrics/flags.go @@ -27,26 +27,26 @@ import ( ) var ( - metricsEnableInfluxDBExportFlag = cli.BoolFlag{ + MetricsEnableInfluxDBExportFlag = cli.BoolFlag{ Name: "metrics.influxdb.export", Usage: "Enable metrics export/push to an external InfluxDB database", } - metricsInfluxDBEndpointFlag = cli.StringFlag{ + MetricsInfluxDBEndpointFlag = cli.StringFlag{ Name: "metrics.influxdb.endpoint", Usage: "Metrics InfluxDB endpoint", Value: "http://127.0.0.1:8086", } - metricsInfluxDBDatabaseFlag = cli.StringFlag{ + MetricsInfluxDBDatabaseFlag = cli.StringFlag{ Name: "metrics.influxdb.database", Usage: "Metrics InfluxDB database", Value: "metrics", } - metricsInfluxDBUsernameFlag = cli.StringFlag{ + MetricsInfluxDBUsernameFlag = cli.StringFlag{ Name: "metrics.influxdb.username", Usage: "Metrics InfluxDB username", Value: "", } - metricsInfluxDBPasswordFlag = cli.StringFlag{ + MetricsInfluxDBPasswordFlag = cli.StringFlag{ Name: "metrics.influxdb.password", Usage: "Metrics InfluxDB password", Value: "", @@ -55,7 +55,7 @@ var ( // It is used so that we can group all nodes and average a measurement across all of them, but also so // that we can select a specific node and inspect its measurements. // https://docs.influxdata.com/influxdb/v1.4/concepts/key_concepts/#tag-key - metricsInfluxDBHostTagFlag = cli.StringFlag{ + MetricsInfluxDBHostTagFlag = cli.StringFlag{ Name: "metrics.influxdb.host.tag", Usage: "Metrics InfluxDB `host` tag attached to all measurements", Value: "localhost", @@ -65,20 +65,24 @@ var ( // Flags holds all command-line flags required for metrics collection. var Flags = []cli.Flag{ utils.MetricsEnabledFlag, - metricsEnableInfluxDBExportFlag, - metricsInfluxDBEndpointFlag, metricsInfluxDBDatabaseFlag, metricsInfluxDBUsernameFlag, metricsInfluxDBPasswordFlag, metricsInfluxDBHostTagFlag, + MetricsEnableInfluxDBExportFlag, + MetricsInfluxDBEndpointFlag, + MetricsInfluxDBDatabaseFlag, + MetricsInfluxDBUsernameFlag, + MetricsInfluxDBPasswordFlag, + MetricsInfluxDBHostTagFlag, } func Setup(ctx *cli.Context) { if gethmetrics.Enabled { log.Info("Enabling swarm metrics collection") var ( - enableExport = ctx.GlobalBool(metricsEnableInfluxDBExportFlag.Name) - endpoint = ctx.GlobalString(metricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(metricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(metricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(metricsInfluxDBPasswordFlag.Name) - hosttag = ctx.GlobalString(metricsInfluxDBHostTagFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBExportFlag.Name) + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + hosttag = ctx.GlobalString(MetricsInfluxDBHostTagFlag.Name) ) // Start system runtime metrics collection diff --git a/swarm/storage/chunker.go b/swarm/storage/chunker.go index 40292e88f9..cbe65372a1 100644 --- a/swarm/storage/chunker.go +++ b/swarm/storage/chunker.go @@ -22,6 +22,7 @@ import ( "fmt" "io" "sync" + "time" "github.com/ethereum/go-ethereum/metrics" ch "github.com/ethereum/go-ethereum/swarm/chunk" @@ -410,10 +411,14 @@ func (r *LazyChunkReader) Size(ctx context.Context, quitC chan bool) (n int64, e log.Debug("lazychunkreader.size", "addr", r.addr) if r.chunkData == nil { + + startTime := time.Now() chunkData, err := r.getter.Get(cctx, Reference(r.addr)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) return 0, err } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) r.chunkData = chunkData s := r.chunkData.Size() log.Debug("lazychunkreader.size", "key", r.addr, "size", s) @@ -542,8 +547,10 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS wg.Add(1) go func(j int64) { childAddress := chunkData[8+j*r.hashSize : 8+(j+1)*r.hashSize] + startTime := time.Now() chunkData, err := r.getter.Get(r.ctx, Reference(childAddress)) if err != nil { + metrics.GetOrRegisterResettingTimer("lcr.getter.get.err", nil).UpdateSince(startTime) log.Debug("lazychunkreader.join", "key", fmt.Sprintf("%x", childAddress), "err", err) select { case errC <- fmt.Errorf("chunk %v-%v not found; key: %s", off, off+treeSize, fmt.Sprintf("%x", childAddress)): @@ -551,6 +558,7 @@ func (r *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, treeS } return } + metrics.GetOrRegisterResettingTimer("lcr.getter.get", nil).UpdateSince(startTime) if l := len(chunkData); l < 9 { select { case errC <- fmt.Errorf("chunk %v-%v incomplete; key: %s, data length %v", off, off+treeSize, fmt.Sprintf("%x", childAddress), l): From 69a8d9841a62af97d98ad59f8c40e34301946b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 13:41:18 +0200 Subject: [PATCH 46/68] cmd/faucet: fix faucet static peer regression --- cmd/faucet/faucet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 2ffe12276e..a7c20db773 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -256,7 +256,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*discv5.Node, network u } for _, boot := range enodes { old, err := enode.ParseV4(boot.String()) - if err != nil { + if err == nil { stack.Server().AddPeer(old) } } From 38c3d88cea61361afe332437581dac8b40adbfca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 13:41:41 +0200 Subject: [PATCH 47/68] cmd/puppeth: support latest docker compose, expose faucet UDP --- cmd/puppeth/module_ethstats.go | 3 ++- cmd/puppeth/module_explorer.go | 1 + cmd/puppeth/module_faucet.go | 4 +++- cmd/puppeth/module_nginx.go | 1 + cmd/puppeth/module_node.go | 1 + cmd/puppeth/module_wallet.go | 1 + 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/module_ethstats.go b/cmd/puppeth/module_ethstats.go index a7d99a2979..58ecb83951 100644 --- a/cmd/puppeth/module_ethstats.go +++ b/cmd/puppeth/module_ethstats.go @@ -43,7 +43,8 @@ version: '2' services: ethstats: build: . - image: {{.Network}}/ethstats{{if not .VHost}} + image: {{.Network}}/ethstats + container_name: {{.Network}}_ethstats_1{{if not .VHost}} ports: - "{{.Port}}:3000"{{end}} environment: diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index e916deaf6f..e465fa04ad 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -77,6 +77,7 @@ services: explorer: build: . image: {{.Network}}/explorer + container_name: {{.Network}}_explorer_1 ports: - "{{.NodePort}}:{{.NodePort}}" - "{{.NodePort}}:{{.NodePort}}/udp"{{if not .VHost}} diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 06c9fc0f58..3a06bf3c68 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -56,8 +56,10 @@ services: faucet: build: . image: {{.Network}}/faucet + container_name: {{.Network}}_faucet_1 ports: - - "{{.EthPort}}:{{.EthPort}}"{{if not .VHost}} + - "{{.EthPort}}:{{.EthPort}}" + - "{{.EthPort}}:{{.EthPort}}/udp"{{if not .VHost}} - "{{.ApiPort}}:8080"{{end}} volumes: - {{.Datadir}}:/root/.faucet diff --git a/cmd/puppeth/module_nginx.go b/cmd/puppeth/module_nginx.go index 7f87661d3b..1b1ae61ff5 100644 --- a/cmd/puppeth/module_nginx.go +++ b/cmd/puppeth/module_nginx.go @@ -40,6 +40,7 @@ services: nginx: build: . image: {{.Network}}/nginx + container_name: {{.Network}}_nginx_1 ports: - "{{.Port}}:80" volumes: diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 069adfe4f4..5d9ef46523 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -55,6 +55,7 @@ services: {{.Type}}: build: . image: {{.Network}}/{{.Type}} + container_name: {{.Network}}_{{.Type}}_1 ports: - "{{.Port}}:{{.Port}}" - "{{.Port}}:{{.Port}}/udp" diff --git a/cmd/puppeth/module_wallet.go b/cmd/puppeth/module_wallet.go index 90812c4a0b..ebaa5b6ae1 100644 --- a/cmd/puppeth/module_wallet.go +++ b/cmd/puppeth/module_wallet.go @@ -57,6 +57,7 @@ services: wallet: build: . image: {{.Network}}/wallet + container_name: {{.Network}}_wallet_1 ports: - "{{.NodePort}}:{{.NodePort}}" - "{{.NodePort}}:{{.NodePort}}/udp" From 83a9a73b899e816a0d1493b7fb1e37abe4bc4566 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 11 Dec 2018 13:19:03 +0100 Subject: [PATCH 48/68] cmd/geth, core, eth: implement Constantinople override flag (#18273) * geth/core/eth: implement constantinople override flag * les: implemnent constantinople override flag for les clients * cmd/geth, eth, les: fix typo, move flag to experimentals --- cmd/geth/config.go | 5 ++++- cmd/geth/main.go | 1 + cmd/utils/flags.go | 6 ++++-- core/genesis.go | 6 ++++++ eth/backend.go | 2 +- eth/config.go | 4 ++++ les/backend.go | 2 +- 7 files changed, 21 insertions(+), 5 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b0749d2329..59f759f0ea 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "math/big" "os" "reflect" "unicode" @@ -152,7 +153,9 @@ func enableWhisper(ctx *cli.Context) bool { func makeFullNode(ctx *cli.Context) *node.Node { stack, cfg := makeConfigNode(ctx) - + if ctx.GlobalIsSet(utils.ConstantinopleOverrideFlag.Name) { + cfg.Eth.ConstantinopleOverride = new(big.Int).SetUint64(ctx.GlobalUint64(utils.ConstantinopleOverrideFlag.Name)) + } utils.RegisterEthService(stack, &cfg.Eth) if ctx.GlobalBool(utils.DashboardEnabledFlag.Name) { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index fc1bf461fb..ebaeba9f46 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -123,6 +123,7 @@ var ( utils.RinkebyFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, + utils.ConstantinopleOverrideFlag, utils.RPCCORSDomainFlag, utils.RPCVirtualHostsFlag, utils.EthStatsURLFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c9115d7a35..60e45d095a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -140,6 +140,10 @@ var ( Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", } + ConstantinopleOverrideFlag = cli.Uint64Flag{ + Name: "override.constantinople", + Usage: "Manually specify constantinople fork-block, overriding the bundled setting", + } DeveloperFlag = cli.BoolFlag{ Name: "dev", Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", @@ -1178,7 +1182,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) } - if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheDatabaseFlag.Name) { cfg.DatabaseCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100 } @@ -1403,7 +1406,6 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chainDb ethdb.Database) { var err error chainDb = MakeChainDatabase(ctx, stack) - config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx)) if err != nil { Fatalf("%v", err) diff --git a/core/genesis.go b/core/genesis.go index 6e71afd61c..c96cb17a36 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -151,6 +151,9 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { + return SetupGenesisBlockWithOverride(db, genesis, nil) +} +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, constantinopleOverride *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -178,6 +181,9 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig // Get the existing chain configuration. newcfg := genesis.configOrDefault(stored) + if constantinopleOverride != nil { + newcfg.ConstantinopleBlock = constantinopleOverride + } storedcfg := rawdb.ReadChainConfig(db, stored) if storedcfg == nil { log.Warn("Found genesis block without chain config") diff --git a/eth/backend.go b/eth/backend.go index 3f538d5f55..354fc17d48 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -118,7 +118,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.ConstantinopleOverride) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/config.go b/eth/config.go index a5c2095124..7c041d1af7 100644 --- a/eth/config.go +++ b/eth/config.go @@ -129,8 +129,12 @@ type Config struct { // Type of the EWASM interpreter ("" for default) EWASMInterpreter string + // Type of the EVM interpreter ("" for default) EVMInterpreter string + + // Constantinople block override (TODO: remove after the fork) + ConstantinopleOverride *big.Int } type configMarshaling struct { diff --git a/les/backend.go b/les/backend.go index a3474a6830..d0db71019e 100644 --- a/les/backend.go +++ b/les/backend.go @@ -82,7 +82,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlock(chainDb, config.Genesis) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.ConstantinopleOverride) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } From 24d727b6d6e2c0cde222fa12155c4a6db5caaf2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 14:20:21 +0200 Subject: [PATCH 49/68] params, swarm: release Geth v1.8.20 and Swarm v0.3.8 --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index 2291d5f3c5..ba9ab202a7 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 20 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 8 // Minor version component of the current release + VersionPatch = 20 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index 57ac05a864..c348109014 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 1e190a3b1cf0444b48cc1aa78367e0d28a561c6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 11 Dec 2018 14:23:57 +0200 Subject: [PATCH 50/68] params, swarm: begin Geth v1.9.0 family, Swarm v0.3.9 cycle --- params/version.go | 8 ++++---- swarm/version/version.go | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/params/version.go b/params/version.go index ba9ab202a7..d3954e0bc3 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 8 // Minor version component of the current release - VersionPatch = 20 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 0 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. diff --git a/swarm/version/version.go b/swarm/version/version.go index c348109014..8301dfd2d4 100644 --- a/swarm/version/version.go +++ b/swarm/version/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From ebbf3dfafb9af891294fa10c04b3c07dd56d8fdc Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Wed, 12 Dec 2018 07:51:29 +0100 Subject: [PATCH 51/68] swarm/shed: add metrics to each shed db (#18277) * swarm/shed: add metrics to each shed db * swarm/shed: push metrics prefix up * swarm/shed: rename prefix to metricsPrefix * swarm/shed: unexport Meter, remove Mutex for quit channel --- swarm/shed/db.go | 205 ++++++++++++++++++++++++++++++- swarm/shed/db_test.go | 6 +- swarm/shed/example_store_test.go | 2 +- 3 files changed, 206 insertions(+), 7 deletions(-) diff --git a/swarm/shed/db.go b/swarm/shed/db.go index e128b8cbc8..7377e12d2d 100644 --- a/swarm/shed/db.go +++ b/swarm/shed/db.go @@ -23,14 +23,23 @@ package shed import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/swarm/log" "github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb/iterator" "github.com/syndtr/goleveldb/leveldb/opt" ) -// The limit for LevelDB OpenFilesCacheCapacity. -const openFileLimit = 128 +const ( + openFileLimit = 128 // The limit for LevelDB OpenFilesCacheCapacity. + writePauseWarningThrottler = 1 * time.Minute +) // DB provides abstractions over LevelDB in order to // implement complex structures using fields and ordered indexes. @@ -38,11 +47,22 @@ const openFileLimit = 128 // information about naming and types. type DB struct { ldb *leveldb.DB + + compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction + compReadMeter metrics.Meter // Meter for measuring the data read during compaction + compWriteMeter metrics.Meter // Meter for measuring the data written during compaction + writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction + writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction + diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read + diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written + + quitChan chan chan error // Quit channel to stop the metrics collection before closing the database } // NewDB constructs a new DB and validates the schema // if it exists in database on the given path. -func NewDB(path string) (db *DB, err error) { +// metricsPrefix is used for metrics collection for the given DB. +func NewDB(path string, metricsPrefix string) (db *DB, err error) { ldb, err := leveldb.OpenFile(path, &opt.Options{ OpenFilesCacheCapacity: openFileLimit, }) @@ -66,6 +86,15 @@ func NewDB(path string) (db *DB, err error) { return nil, err } } + + // Configure meters for DB + db.configure(metricsPrefix) + + // Create a quit channel for the periodic metrics collector and run it + db.quitChan = make(chan chan error) + + go db.meter(10 * time.Second) + return db, nil } @@ -126,5 +155,175 @@ func (db *DB) WriteBatch(batch *leveldb.Batch) (err error) { // Close closes LevelDB database. func (db *DB) Close() (err error) { + close(db.quitChan) return db.ldb.Close() } + +// Configure configures the database metrics collectors +func (db *DB) configure(prefix string) { + // Initialize all the metrics collector at the requested prefix + db.compTimeMeter = metrics.NewRegisteredMeter(prefix+"compact/time", nil) + db.compReadMeter = metrics.NewRegisteredMeter(prefix+"compact/input", nil) + db.compWriteMeter = metrics.NewRegisteredMeter(prefix+"compact/output", nil) + db.diskReadMeter = metrics.NewRegisteredMeter(prefix+"disk/read", nil) + db.diskWriteMeter = metrics.NewRegisteredMeter(prefix+"disk/write", nil) + db.writeDelayMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/duration", nil) + db.writeDelayNMeter = metrics.NewRegisteredMeter(prefix+"compact/writedelay/counter", nil) +} + +func (db *DB) meter(refresh time.Duration) { + // Create the counters to store current and previous compaction values + compactions := make([][]float64, 2) + for i := 0; i < 2; i++ { + compactions[i] = make([]float64, 3) + } + // Create storage for iostats. + var iostats [2]float64 + + // Create storage and warning log tracer for write delay. + var ( + delaystats [2]int64 + lastWritePaused time.Time + ) + + var ( + errc chan error + merr error + ) + + // Iterate ad infinitum and collect the stats + for i := 1; errc == nil && merr == nil; i++ { + // Retrieve the database stats + stats, err := db.ldb.GetProperty("leveldb.stats") + if err != nil { + log.Error("Failed to read database stats", "err", err) + merr = err + continue + } + // Find the compaction table, skip the header + lines := strings.Split(stats, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { + lines = lines[1:] + } + if len(lines) <= 3 { + log.Error("Compaction table not found") + merr = errors.New("compaction table not found") + continue + } + lines = lines[3:] + + // Iterate over all the table rows, and accumulate the entries + for j := 0; j < len(compactions[i%2]); j++ { + compactions[i%2][j] = 0 + } + for _, line := range lines { + parts := strings.Split(line, "|") + if len(parts) != 6 { + break + } + for idx, counter := range parts[3:] { + value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) + if err != nil { + log.Error("Compaction entry parsing failed", "err", err) + merr = err + continue + } + compactions[i%2][idx] += value + } + } + // Update all the requested meters + if db.compTimeMeter != nil { + db.compTimeMeter.Mark(int64((compactions[i%2][0] - compactions[(i-1)%2][0]) * 1000 * 1000 * 1000)) + } + if db.compReadMeter != nil { + db.compReadMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1024 * 1024)) + } + if db.compWriteMeter != nil { + db.compWriteMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) + } + + // Retrieve the write delay statistic + writedelay, err := db.ldb.GetProperty("leveldb.writedelay") + if err != nil { + log.Error("Failed to read database write delay statistic", "err", err) + merr = err + continue + } + var ( + delayN int64 + delayDuration string + duration time.Duration + paused bool + ) + if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { + log.Error("Write delay statistic not found") + merr = err + continue + } + duration, err = time.ParseDuration(delayDuration) + if err != nil { + log.Error("Failed to parse delay duration", "err", err) + merr = err + continue + } + if db.writeDelayNMeter != nil { + db.writeDelayNMeter.Mark(delayN - delaystats[0]) + } + if db.writeDelayMeter != nil { + db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) + } + // If a warning that db is performing compaction has been displayed, any subsequent + // warnings will be withheld for one minute not to overwhelm the user. + if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && + time.Now().After(lastWritePaused.Add(writePauseWarningThrottler)) { + log.Warn("Database compacting, degraded performance") + lastWritePaused = time.Now() + } + delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() + + // Retrieve the database iostats. + ioStats, err := db.ldb.GetProperty("leveldb.iostats") + if err != nil { + log.Error("Failed to read database iostats", "err", err) + merr = err + continue + } + var nRead, nWrite float64 + parts := strings.Split(ioStats, " ") + if len(parts) < 2 { + log.Error("Bad syntax of ioStats", "ioStats", ioStats) + merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) + continue + } + if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { + log.Error("Bad syntax of read entry", "entry", parts[0]) + merr = err + continue + } + if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { + log.Error("Bad syntax of write entry", "entry", parts[1]) + merr = err + continue + } + if db.diskReadMeter != nil { + db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) + } + if db.diskWriteMeter != nil { + db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) + } + iostats[0], iostats[1] = nRead, nWrite + + // Sleep a bit, then repeat the stats collection + select { + case errc = <-db.quitChan: + // Quit requesting, stop hammering the database + case <-time.After(refresh): + // Timeout, gather a new set of stats + } + } + + if errc == nil { + errc = <-db.quitChan + } + errc <- merr +} diff --git a/swarm/shed/db_test.go b/swarm/shed/db_test.go index 45325beeb8..65fdac4a61 100644 --- a/swarm/shed/db_test.go +++ b/swarm/shed/db_test.go @@ -55,7 +55,7 @@ func TestDB_persistence(t *testing.T) { } defer os.RemoveAll(dir) - db, err := NewDB(dir) + db, err := NewDB(dir, "") if err != nil { t.Fatal(err) } @@ -73,7 +73,7 @@ func TestDB_persistence(t *testing.T) { t.Fatal(err) } - db2, err := NewDB(dir) + db2, err := NewDB(dir, "") if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func newTestDB(t *testing.T) (db *DB, cleanupFunc func()) { t.Fatal(err) } cleanupFunc = func() { os.RemoveAll(dir) } - db, err = NewDB(dir) + db, err = NewDB(dir, "") if err != nil { cleanupFunc() t.Fatal(err) diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index 2ed0be1413..908a1e446d 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -52,7 +52,7 @@ type Store struct { // and possible conflicts with schema from existing database is checked // automatically. func New(path string) (s *Store, err error) { - db, err := shed.NewDB(path) + db, err := shed.NewDB(path, "") if err != nil { return nil, err } From 4e6f53ac3360c4e90fdb419e7073e4316b2a49f7 Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Wed, 12 Dec 2018 16:22:17 +0100 Subject: [PATCH 52/68] swarm/storage: simplify ChunkValidator interface (#18285) --- swarm/storage/feed/handler.go | 11 +++++------ swarm/storage/feed/handler_test.go | 4 ++-- swarm/storage/feed/request.go | 6 ++++-- swarm/storage/feed/request_test.go | 6 +++--- swarm/storage/localstore.go | 2 +- swarm/storage/localstore_test.go | 2 +- swarm/storage/types.go | 7 ++++--- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/swarm/storage/feed/handler.go b/swarm/storage/feed/handler.go index 9e2640282c..33542b6e43 100644 --- a/swarm/storage/feed/handler.go +++ b/swarm/storage/feed/handler.go @@ -82,9 +82,8 @@ func (h *Handler) SetStore(store *storage.NetStore) { // Validate is a chunk validation method // If it looks like a feed update, the chunk address is checked against the userAddr of the update's signature // It implements the storage.ChunkValidator interface -func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { - dataLength := len(data) - if dataLength < minimumSignedUpdateLength { +func (h *Handler) Validate(chunk storage.Chunk) bool { + if len(chunk.Data()) < minimumSignedUpdateLength { return false } @@ -94,8 +93,8 @@ func (h *Handler) Validate(chunkAddr storage.Address, data []byte) bool { // First, deserialize the chunk var r Request - if err := r.fromChunk(chunkAddr, data); err != nil { - log.Debug("Invalid feed update chunk", "addr", chunkAddr.Hex(), "err", err.Error()) + if err := r.fromChunk(chunk); err != nil { + log.Debug("Invalid feed update chunk", "addr", chunk.Address(), "err", err) return false } @@ -198,7 +197,7 @@ func (h *Handler) Lookup(ctx context.Context, query *Query) (*cacheEntry, error) } var request Request - if err := request.fromChunk(chunk.Address(), chunk.Data()); err != nil { + if err := request.fromChunk(chunk); err != nil { return nil, nil } if request.Time <= timeLimit { diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index fb2ef3a6ba..9c39215c56 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -366,7 +366,7 @@ func TestValidator(t *testing.T) { if err != nil { t.Fatal(err) } - if !rh.Validate(chunk.Address(), chunk.Data()) { + if !rh.Validate(chunk) { t.Fatal("Chunk validator fail on update chunk") } @@ -375,7 +375,7 @@ func TestValidator(t *testing.T) { address[0] = 11 address[15] = 99 - if rh.Validate(address, chunk.Data()) { + if rh.Validate(storage.NewChunk(address, chunk.Data())) { t.Fatal("Expected Validate to fail with false chunk address") } } diff --git a/swarm/storage/feed/request.go b/swarm/storage/feed/request.go index 6968d8b9ad..dd91a7cf45 100644 --- a/swarm/storage/feed/request.go +++ b/swarm/storage/feed/request.go @@ -171,9 +171,11 @@ func (r *Request) toChunk() (storage.Chunk, error) { } // fromChunk populates this structure from chunk data. It does not verify the signature is valid. -func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error { +func (r *Request) fromChunk(chunk storage.Chunk) error { // for update chunk layout see Request definition + chunkdata := chunk.Data() + //deserialize the feed update portion if err := r.Update.binaryGet(chunkdata[:len(chunkdata)-signatureLength]); err != nil { return err @@ -189,7 +191,7 @@ func (r *Request) fromChunk(updateAddr storage.Address, chunkdata []byte) error } r.Signature = signature - r.idAddr = updateAddr + r.idAddr = chunk.Address() r.binaryData = chunkdata return nil diff --git a/swarm/storage/feed/request_test.go b/swarm/storage/feed/request_test.go index f5de32b74f..c30158fddf 100644 --- a/swarm/storage/feed/request_test.go +++ b/swarm/storage/feed/request_test.go @@ -197,7 +197,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { // Test that parseUpdate fails if the chunk is too small var r Request - if err := r.fromChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength)); err == nil { + if err := r.fromChunk(storage.NewChunk(storage.ZeroAddr, make([]byte, minimumUpdateDataLength-1+signatureLength))); err == nil { t.Fatalf("Expected request.fromChunk to fail when chunkData contains less than %d bytes", minimumUpdateDataLength) } @@ -226,7 +226,7 @@ func TestUpdateChunkSerializationErrorChecking(t *testing.T) { compareByteSliceToExpectedHex(t, "chunk", chunk.Data(), "0x0000000000000000776f726c64206e657773207265706f72742c20657665727920686f7572000000876a8936a7cd0b79ef0735ad0896c1afe278781ce803000000000019416c206269656e206861636572206a616dc3a173206c652066616c7461207072656d696f5a0ffe0bc27f207cd5b00944c8b9cee93e08b89b5ada777f123ac535189333f174a6a4ca2f43a92c4a477a49d774813c36ce8288552c58e6205b0ac35d0507eb00") var recovered Request - recovered.fromChunk(chunk.Address(), chunk.Data()) + recovered.fromChunk(chunk) if !reflect.DeepEqual(recovered, r) { t.Fatal("Expected recovered feed update request to equal the original one") } @@ -282,7 +282,7 @@ func TestReverse(t *testing.T) { // check that we can recover the owner account from the update chunk's signature var checkUpdate Request - if err := checkUpdate.fromChunk(chunk.Address(), chunk.Data()); err != nil { + if err := checkUpdate.fromChunk(chunk); err != nil { t.Fatal(err) } checkdigest, err := checkUpdate.GetDigest() diff --git a/swarm/storage/localstore.go b/swarm/storage/localstore.go index 111821ff65..956560902b 100644 --- a/swarm/storage/localstore.go +++ b/swarm/storage/localstore.go @@ -92,7 +92,7 @@ func (ls *LocalStore) isValid(chunk Chunk) bool { // ls.Validators contains a list of one validator per chunk type. // if one validator succeeds, then the chunk is valid for _, v := range ls.Validators { - if valid = v.Validate(chunk.Address(), chunk.Data()); valid { + if valid = v.Validate(chunk); valid { break } } diff --git a/swarm/storage/localstore_test.go b/swarm/storage/localstore_test.go index 7a07726d1c..7a4162a475 100644 --- a/swarm/storage/localstore_test.go +++ b/swarm/storage/localstore_test.go @@ -118,7 +118,7 @@ func TestValidator(t *testing.T) { type boolTestValidator bool -func (self boolTestValidator) Validate(addr Address, data []byte) bool { +func (self boolTestValidator) Validate(chunk Chunk) bool { return bool(self) } diff --git a/swarm/storage/types.go b/swarm/storage/types.go index 42557766ef..e5c5f89a09 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -327,7 +327,7 @@ func (c ChunkData) Data() []byte { } type ChunkValidator interface { - Validate(addr Address, data []byte) bool + Validate(chunk Chunk) bool } // Provides method for validation of content address in chunks @@ -344,7 +344,8 @@ func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator { } // Validate that the given key is a valid content address for the given data -func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { +func (v *ContentAddressValidator) Validate(chunk Chunk) bool { + data := chunk.Data() if l := len(data); l < 9 || l > ch.DefaultSize+8 { // log.Error("invalid chunk size", "chunk", addr.Hex(), "size", l) return false @@ -355,7 +356,7 @@ func (v *ContentAddressValidator) Validate(addr Address, data []byte) bool { hasher.Write(data[8:]) hash := hasher.Sum(nil) - return bytes.Equal(hash, addr[:]) + return bytes.Equal(hash, chunk.Address()) } type ChunkStore interface { From b3be9b7cd8f5f0e828d4d2aa6db1216ccd32ce3a Mon Sep 17 00:00:00 2001 From: Corey Lin <514971757@qq.com> Date: Thu, 13 Dec 2018 17:21:52 +0800 Subject: [PATCH 53/68] usbwallet: check returned error when decoding hexstr (#18056) * usbwallet: check returned error when decoding hexstr * Update accounts/usbwallet/ledger.go Co-Authored-By: CoreyLin <514971757@qq.com> * usbwallet: check hex decode error --- accounts/usbwallet/ledger.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 7d5f67908d..c30903b5b7 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -257,7 +257,9 @@ func (w *ledgerDriver) ledgerDerive(derivationPath []uint32) (common.Address, er // Decode the hex sting into an Ethereum address and return var address common.Address - hex.Decode(address[:], hexstr) + if _, err = hex.Decode(address[:], hexstr); err != nil { + return common.Address{}, err + } return address, nil } From e57e4571d3382c504a01f87be654d984f26e2dd1 Mon Sep 17 00:00:00 2001 From: Shuai Qi Date: Thu, 13 Dec 2018 17:25:13 +0800 Subject: [PATCH 54/68] crypto/secp256k1: Fix invalid document link (#18297) --- crypto/secp256k1/curve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index 56be235b36..5409ee1d2c 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -310,7 +310,7 @@ var theCurve = new(BitCurve) func init() { // See SEC 2 section 2.7.1 // curve parameters taken from: - // http://www.secg.org/collateral/sec2_final.pdf + // http://www.secg.org/sec2-v2.pdf theCurve.P, _ = new(big.Int).SetString("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 0) theCurve.N, _ = new(big.Int).SetString("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 0) theCurve.B, _ = new(big.Int).SetString("0x0000000000000000000000000000000000000000000000000000000000000007", 0) From e79821cabeda5c6907f03073d11fe6d83367f494 Mon Sep 17 00:00:00 2001 From: tzapu Date: Thu, 13 Dec 2018 16:12:19 +0200 Subject: [PATCH 55/68] accounts/abi: argument type and name were reversed (#17947) argument type and name were reversed --- accounts/abi/event.go | 6 ++-- accounts/abi/event_test.go | 41 ++++++++++++++++++++++--- accounts/abi/method.go | 6 ++-- accounts/abi/method_test.go | 61 +++++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 accounts/abi/method_test.go diff --git a/accounts/abi/event.go b/accounts/abi/event.go index a3f6be9732..9392c1990c 100644 --- a/accounts/abi/event.go +++ b/accounts/abi/event.go @@ -36,12 +36,12 @@ type Event struct { func (e Event) String() string { inputs := make([]string, len(e.Inputs)) for i, input := range e.Inputs { - inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name) if input.Indexed { - inputs[i] = fmt.Sprintf("%v indexed %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v indexed %v", input.Type, input.Name) } } - return fmt.Sprintf("e %v(%v)", e.Name, strings.Join(inputs, ", ")) + return fmt.Sprintf("event %v(%v)", e.Name, strings.Join(inputs, ", ")) } // Id returns the canonical representation of the event's signature used by the diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 3bfdd7c0ab..e735cceb88 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -87,12 +87,12 @@ func TestEventId(t *testing.T) { }{ { definition: `[ - { "type" : "event", "name" : "balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, - { "type" : "event", "name" : "check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] } + { "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, + { "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] } ]`, expectations: map[string]common.Hash{ - "balance": crypto.Keccak256Hash([]byte("balance(uint256)")), - "check": crypto.Keccak256Hash([]byte("check(address,uint256)")), + "Balance": crypto.Keccak256Hash([]byte("Balance(uint256)")), + "Check": crypto.Keccak256Hash([]byte("Check(address,uint256)")), }, }, } @@ -111,6 +111,39 @@ func TestEventId(t *testing.T) { } } +func TestEventString(t *testing.T) { + var table = []struct { + definition string + expectations map[string]string + }{ + { + definition: `[ + { "type" : "event", "name" : "Balance", "inputs": [{ "name" : "in", "type": "uint256" }] }, + { "type" : "event", "name" : "Check", "inputs": [{ "name" : "t", "type": "address" }, { "name": "b", "type": "uint256" }] }, + { "type" : "event", "name" : "Transfer", "inputs": [{ "name": "from", "type": "address", "indexed": true }, { "name": "to", "type": "address", "indexed": true }, { "name": "value", "type": "uint256" }] } + ]`, + expectations: map[string]string{ + "Balance": "event Balance(uint256 in)", + "Check": "event Check(address t, uint256 b)", + "Transfer": "event Transfer(address indexed from, address indexed to, uint256 value)", + }, + }, + } + + for _, test := range table { + abi, err := JSON(strings.NewReader(test.definition)) + if err != nil { + t.Fatal(err) + } + + for name, event := range abi.Events { + if event.String() != test.expectations[name] { + t.Errorf("expected string to be %s, got %s", test.expectations[name], event.String()) + } + } + } +} + // TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array. func TestEventMultiValueWithArrayUnpack(t *testing.T) { definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` diff --git a/accounts/abi/method.go b/accounts/abi/method.go index 5831057652..2d8d3d6589 100644 --- a/accounts/abi/method.go +++ b/accounts/abi/method.go @@ -56,14 +56,14 @@ func (method Method) Sig() string { func (method Method) String() string { inputs := make([]string, len(method.Inputs)) for i, input := range method.Inputs { - inputs[i] = fmt.Sprintf("%v %v", input.Name, input.Type) + inputs[i] = fmt.Sprintf("%v %v", input.Type, input.Name) } outputs := make([]string, len(method.Outputs)) for i, output := range method.Outputs { + outputs[i] = output.Type.String() if len(output.Name) > 0 { - outputs[i] = fmt.Sprintf("%v ", output.Name) + outputs[i] += fmt.Sprintf(" %v", output.Name) } - outputs[i] += output.Type.String() } constant := "" if method.Const { diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go new file mode 100644 index 0000000000..a98f1cd31f --- /dev/null +++ b/accounts/abi/method_test.go @@ -0,0 +1,61 @@ +// Copyright 2016 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package abi + +import ( + "strings" + "testing" +) + +const methoddata = ` +[ + { "type" : "function", "name" : "balance", "constant" : true }, + { "type" : "function", "name" : "send", "constant" : false, "inputs" : [ { "name" : "amount", "type" : "uint256" } ] }, + { "type" : "function", "name" : "transfer", "constant" : false, "inputs" : [ { "name" : "from", "type" : "address" }, { "name" : "to", "type" : "address" }, { "name" : "value", "type" : "uint256" } ], "outputs" : [ { "name" : "success", "type" : "bool" } ] } +]` + +func TestMethodString(t *testing.T) { + var table = []struct { + method string + expectation string + }{ + { + method: "balance", + expectation: "function balance() constant returns()", + }, + { + method: "send", + expectation: "function send(uint256 amount) returns()", + }, + { + method: "transfer", + expectation: "function transfer(address from, address to, uint256 value) returns(bool success)", + }, + } + + abi, err := JSON(strings.NewReader(methoddata)) + if err != nil { + t.Fatal(err) + } + + for _, test := range table { + got := abi.Methods[test.method].String() + if got != test.expectation { + t.Errorf("expected string to be %s, got %s", test.expectation, got) + } + } +} From 3796751efcc3fab71799450f1bff370d081438ec Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 14 Dec 2018 11:08:11 +0100 Subject: [PATCH 56/68] rpc: add application/json-rpc as accepted content type, fixes #18293 (#18310) --- rpc/http.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/rpc/http.go b/rpc/http.go index af79858e2b..674166fb3a 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -36,11 +36,15 @@ import ( ) const ( - contentType = "application/json" maxRequestContentLength = 1024 * 512 ) -var nullAddr, _ = net.ResolveTCPAddr("tcp", "127.0.0.1:0") +var ( + // https://www.jsonrpc.org/historical/json-rpc-over-http.html#id13 + acceptedContentTypes = []string{"application/json", "application/json-rpc", "application/jsonrequest"} + contentType = acceptedContentTypes[0] + nullAddr, _ = net.ResolveTCPAddr("tcp", "127.0.0.1:0") +) type httpConn struct { client *http.Client @@ -263,12 +267,21 @@ func validateRequest(r *http.Request) (int, error) { err := fmt.Errorf("content length too large (%d>%d)", r.ContentLength, maxRequestContentLength) return http.StatusRequestEntityTooLarge, err } - mt, _, err := mime.ParseMediaType(r.Header.Get("content-type")) - if r.Method != http.MethodOptions && (err != nil || mt != contentType) { - err := fmt.Errorf("invalid content type, only %s is supported", contentType) - return http.StatusUnsupportedMediaType, err + // Allow OPTIONS (regardless of content-type) + if r.Method == http.MethodOptions { + return 0, nil } - return 0, nil + // Check content-type + if mt, _, err := mime.ParseMediaType(r.Header.Get("content-type")); err == nil { + for _, accepted := range acceptedContentTypes { + if accepted == mt { + return 0, nil + } + } + } + // Invalid content-type + err := fmt.Errorf("invalid content type, only %s is supported", contentType) + return http.StatusUnsupportedMediaType, err } func newCorsHandler(srv *Server, allowedOrigins []string) http.Handler { From 9940d93a43bab1f060e8fbfadff510c3298f2726 Mon Sep 17 00:00:00 2001 From: qd-ethan <31876119+qdgogogo@users.noreply.github.com> Date: Fri, 14 Dec 2018 18:15:31 +0800 Subject: [PATCH 57/68] Comment error (#18303) --- core/tx_cacher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_cacher.go b/core/tx_cacher.go index bcaa5ead38..b1e5d676a2 100644 --- a/core/tx_cacher.go +++ b/core/tx_cacher.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// senderCacher is a concurrent transaction sender recoverer anc cacher. +// senderCacher is a concurrent transaction sender recoverer and cacher. var senderCacher = newTxSenderCacher(runtime.NumCPU()) // txSenderCacherRequest is a request for recovering transaction senders with a From 88d7d4fed4f95df9b81fa6acc66c760e0cb08cd7 Mon Sep 17 00:00:00 2001 From: Adam Schmideg Date: Fri, 14 Dec 2018 14:50:10 +0100 Subject: [PATCH 58/68] Change issue labels in bot configs to the new prefixed version --- .github/no-response.yml | 2 +- .github/stale.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/no-response.yml b/.github/no-response.yml index b6e96efdc6..903d4ce85f 100644 --- a/.github/no-response.yml +++ b/.github/no-response.yml @@ -1,7 +1,7 @@ # Number of days of inactivity before an Issue is closed for lack of response daysUntilClose: 30 # Label requiring a response -responseRequiredLabel: more-information-needed +responseRequiredLabel: "need:more-information" # Comment to post when closing an Issue for lack of response. Set to `false` to disable closeComment: > This issue has been automatically closed because there has been no response diff --git a/.github/stale.yml b/.github/stale.yml index c621939c3f..6d921cc795 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -7,7 +7,7 @@ exemptLabels: - pinned - security # Label to use when marking an issue as stale -staleLabel: stale +staleLabel: "status:inactive" # Comment to post when marking an issue as stale. Set to `false` to disable markComment: > This issue has been automatically marked as stale because it has not had From fe26b2f366693bfd2cff71e81fa5512977affd7b Mon Sep 17 00:00:00 2001 From: Shuai Qi Date: Fri, 14 Dec 2018 21:55:03 +0800 Subject: [PATCH 59/68] core/state: rename 'new' variable (#18301) --- core/state/statedb.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 76e67d839d..2230b10ef0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -468,9 +468,9 @@ func (self *StateDB) createObject(addr common.Address) (newobj, prev *stateObjec // // Carrying over the balance ensures that Ether doesn't disappear. func (self *StateDB) CreateAccount(addr common.Address) { - new, prev := self.createObject(addr) + newObj, prev := self.createObject(addr) if prev != nil { - new.setBalance(prev.data.Balance) + newObj.setBalance(prev.data.Balance) } } From aad3c67a92cd4f3cc3a885fdc514ba2a7fb3e0a3 Mon Sep 17 00:00:00 2001 From: yahtoo Date: Fri, 14 Dec 2018 21:55:51 +0800 Subject: [PATCH 60/68] p2p/discv5: don't hash findnode target in lookup against table (#18309) --- p2p/discv5/net.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/discv5/net.go b/p2p/discv5/net.go index cdeb28dd5b..55e596755b 100644 --- a/p2p/discv5/net.go +++ b/p2p/discv5/net.go @@ -800,7 +800,7 @@ func (n *nodeNetGuts) startNextQuery(net *Network) { func (q *findnodeQuery) start(net *Network) bool { // Satisfy queries against the local node directly. if q.remote == net.tab.self { - closest := net.tab.closest(crypto.Keccak256Hash(q.target[:]), bucketSize) + closest := net.tab.closest(q.target, bucketSize) q.reply <- closest.entries return true } From aebf9e2fe7beb6d23db94d0885cc7a1089b15d87 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 14 Dec 2018 15:05:22 +0100 Subject: [PATCH 61/68] .github: add @gballet as abi codeowner (#18306) --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9a61d39325..11c4dcedc2 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,6 +2,7 @@ # Each line is a file pattern followed by one or more owners. accounts/usbwallet @karalabe +accounts/abi @gballet consensus @karalabe core/ @karalabe @holiman eth/ @karalabe From 3ad73443c7a96c928577234df5c742d53f3293ef Mon Sep 17 00:00:00 2001 From: weimumu <934657014@qq.com> Date: Mon, 17 Dec 2018 16:50:52 +0800 Subject: [PATCH 62/68] fix slice unpack bug in accounts/abi (#18321) * fix slice unpack bug in accounts/abi --- accounts/abi/unpack.go | 2 +- accounts/abi/unpack_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index d5875140cc..c3bbbb251a 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -151,7 +151,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error) // Arrays have packed elements, resulting in longer unpack steps. // Slices have just 32 bytes per element (pointing to the contents). elemSize := 32 - if t.T == ArrayTy { + if t.T == ArrayTy || t.T == SliceTy { elemSize = getFullElemSize(t.Elem) } diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index 97552b90cf..29ed4522cf 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -191,6 +191,11 @@ var unpackTests = []unpackTest{ enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", want: [][2]uint8{{1, 2}}, }, + { + def: `[{"type": "uint8[2][]"}]`, + enc: "000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", + want: [][2]uint8{{1, 2}, {1, 2}}, + }, { def: `[{"type": "uint16[]"}]`, enc: "0000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000002", From d322c9d5504ad3fd4dfeb9dff29e92b35856042f Mon Sep 17 00:00:00 2001 From: Anton Evangelatov Date: Mon, 17 Dec 2018 11:32:55 +0100 Subject: [PATCH 63/68] swarm/storage/feed: remove unused code (#18324) --- swarm/storage/feed/handler.go | 11 ++++------- swarm/storage/feed/handler_test.go | 1 - swarm/storage/feed/timestampprovider.go | 22 ---------------------- 3 files changed, 4 insertions(+), 30 deletions(-) diff --git a/swarm/storage/feed/handler.go b/swarm/storage/feed/handler.go index 33542b6e43..063d3e92a3 100644 --- a/swarm/storage/feed/handler.go +++ b/swarm/storage/feed/handler.go @@ -23,7 +23,6 @@ import ( "context" "fmt" "sync" - "time" "github.com/ethereum/go-ethereum/swarm/storage/feed/lookup" @@ -32,12 +31,10 @@ import ( ) type Handler struct { - chunkStore *storage.NetStore - HashSize int - cache map[uint64]*cacheEntry - cacheLock sync.RWMutex - storeTimeout time.Duration - queryMaxPeriods uint32 + chunkStore *storage.NetStore + HashSize int + cache map[uint64]*cacheEntry + cacheLock sync.RWMutex } // HandlerParams pass parameters to the Handler constructor NewHandler diff --git a/swarm/storage/feed/handler_test.go b/swarm/storage/feed/handler_test.go index 9c39215c56..2f8a524535 100644 --- a/swarm/storage/feed/handler_test.go +++ b/swarm/storage/feed/handler_test.go @@ -40,7 +40,6 @@ var ( } cleanF func() subtopicName = "føø.bar" - hashfunc = storage.MakeHashFunc(storage.DefaultHash) ) func init() { diff --git a/swarm/storage/feed/timestampprovider.go b/swarm/storage/feed/timestampprovider.go index 072dc3a48d..fb60cea9c3 100644 --- a/swarm/storage/feed/timestampprovider.go +++ b/swarm/storage/feed/timestampprovider.go @@ -17,7 +17,6 @@ package feed import ( - "encoding/binary" "encoding/json" "time" ) @@ -30,32 +29,11 @@ type Timestamp struct { Time uint64 `json:"time"` // Unix epoch timestamp, in seconds } -// 8 bytes uint64 Time -const timestampLength = 8 - // timestampProvider interface describes a source of timestamp information type timestampProvider interface { Now() Timestamp // returns the current timestamp information } -// binaryGet populates the timestamp structure from the given byte slice -func (t *Timestamp) binaryGet(data []byte) error { - if len(data) != timestampLength { - return NewError(ErrCorruptData, "timestamp data has the wrong size") - } - t.Time = binary.LittleEndian.Uint64(data[:8]) - return nil -} - -// binaryPut Serializes a Timestamp to a byte slice -func (t *Timestamp) binaryPut(data []byte) error { - if len(data) != timestampLength { - return NewError(ErrCorruptData, "timestamp data has the wrong size") - } - binary.LittleEndian.PutUint64(data, t.Time) - return nil -} - // UnmarshalJSON implements the json.Unmarshaller interface func (t *Timestamp) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &t.Time) From 472c23a8015cd84b193d2d0efb4592a664de3c62 Mon Sep 17 00:00:00 2001 From: Elad Date: Mon, 17 Dec 2018 16:49:01 +0530 Subject: [PATCH 64/68] p2p/simulation: move connection methods from swarm/network/simulation (#18323) --- .../simulation => p2p/simulations}/connect.go | 119 ++++--- p2p/simulations/connect_test.go | 190 +++++++++++ p2p/simulations/network.go | 83 ++++- p2p/simulations/test.go | 134 ++++++++ swarm/network/simulation/connect_test.go | 306 ------------------ swarm/network/simulation/events_test.go | 2 +- swarm/network/simulation/node.go | 64 +--- swarm/network/simulation/node_test.go | 9 +- swarm/network/simulation/service.go | 2 +- swarm/network/simulation/simulation.go | 1 - swarm/network/stream/delivery_test.go | 7 +- .../network/stream/snapshot_retrieval_test.go | 6 +- swarm/network/stream/snapshot_sync_test.go | 14 +- .../visualized_snapshot_sync_sim_test.go | 2 +- 14 files changed, 500 insertions(+), 439 deletions(-) rename {swarm/network/simulation => p2p/simulations}/connect.go (51%) create mode 100644 p2p/simulations/connect_test.go create mode 100644 p2p/simulations/test.go delete mode 100644 swarm/network/simulation/connect_test.go diff --git a/swarm/network/simulation/connect.go b/p2p/simulations/connect.go similarity index 51% rename from swarm/network/simulation/connect.go rename to p2p/simulations/connect.go index 8b2aa1bfa7..606dda0567 100644 --- a/swarm/network/simulation/connect.go +++ b/p2p/simulations/connect.go @@ -14,65 +14,69 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package simulation +package simulations import ( + "errors" "strings" "github.com/ethereum/go-ethereum/p2p/enode" ) +var ( + ErrNodeNotFound = errors.New("node not found") + ErrNoPivotNode = errors.New("no pivot node set") +) + // ConnectToPivotNode connects the node with provided NodeID -// to the pivot node, already set by Simulation.SetPivotNode method. +// to the pivot node, already set by Network.SetPivotNode method. // It is useful when constructing a star network topology -// when simulation adds and removes nodes dynamically. -func (s *Simulation) ConnectToPivotNode(id enode.ID) (err error) { - pid := s.PivotNodeID() - if pid == nil { +// when Network adds and removes nodes dynamically. +func (net *Network) ConnectToPivotNode(id enode.ID) (err error) { + pivot := net.GetPivotNode() + if pivot == nil { return ErrNoPivotNode } - return s.connect(*pid, id) + return net.connect(pivot.ID(), id) } // ConnectToLastNode connects the node with provided NodeID // to the last node that is up, and avoiding connection to self. // It is useful when constructing a chain network topology -// when simulation adds and removes nodes dynamically. -func (s *Simulation) ConnectToLastNode(id enode.ID) (err error) { - ids := s.UpNodeIDs() +// when Network adds and removes nodes dynamically. +func (net *Network) ConnectToLastNode(id enode.ID) (err error) { + ids := net.getUpNodeIDs() l := len(ids) if l < 2 { return nil } - lid := ids[l-1] - if lid == id { - lid = ids[l-2] + last := ids[l-1] + if last == id { + last = ids[l-2] } - return s.connect(lid, id) + return net.connect(last, id) } -// ConnectToRandomNode connects the node with provieded NodeID +// ConnectToRandomNode connects the node with provided NodeID // to a random node that is up. -func (s *Simulation) ConnectToRandomNode(id enode.ID) (err error) { - n := s.RandomUpNode(id) - if n == nil { +func (net *Network) ConnectToRandomNode(id enode.ID) (err error) { + selected := net.GetRandomUpNode(id) + if selected == nil { return ErrNodeNotFound } - return s.connect(n.ID, id) + return net.connect(selected.ID(), id) } // ConnectNodesFull connects all nodes one to another. // It provides a complete connectivity in the network // which should be rarely needed. -func (s *Simulation) ConnectNodesFull(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesFull(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } - l := len(ids) - for i := 0; i < l; i++ { - for j := i + 1; j < l; j++ { - err = s.connect(ids[i], ids[j]) - if err != nil { + for i, lid := range ids { + for _, rid := range ids[i+1:] { + if err = net.connect(lid, rid); err != nil { return err } } @@ -82,14 +86,13 @@ func (s *Simulation) ConnectNodesFull(ids []enode.ID) (err error) { // ConnectNodesChain connects all nodes in a chain topology. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesChain(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesChain(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } l := len(ids) for i := 0; i < l-1; i++ { - err = s.connect(ids[i], ids[i+1]) - if err != nil { + if err := net.connect(ids[i], ids[i+1]); err != nil { return err } } @@ -98,37 +101,32 @@ func (s *Simulation) ConnectNodesChain(ids []enode.ID) (err error) { // ConnectNodesRing connects all nodes in a ring topology. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesRing(ids []enode.ID) (err error) { +func (net *Network) ConnectNodesRing(ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } l := len(ids) if l < 2 { return nil } - for i := 0; i < l-1; i++ { - err = s.connect(ids[i], ids[i+1]) - if err != nil { - return err - } + if err := net.ConnectNodesChain(ids); err != nil { + return err } - return s.connect(ids[l-1], ids[0]) + return net.connect(ids[l-1], ids[0]) } // ConnectNodesStar connects all nodes in a star topology // with the center at provided NodeID. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesStar(id enode.ID, ids []enode.ID) (err error) { +func (net *Network) ConnectNodesStar(pivot enode.ID, ids []enode.ID) (err error) { if ids == nil { - ids = s.UpNodeIDs() + ids = net.getUpNodeIDs() } - l := len(ids) - for i := 0; i < l; i++ { - if id == ids[i] { + for _, id := range ids { + if pivot == id { continue } - err = s.connect(id, ids[i]) - if err != nil { + if err := net.connect(pivot, id); err != nil { return err } } @@ -138,17 +136,17 @@ func (s *Simulation) ConnectNodesStar(id enode.ID, ids []enode.ID) (err error) { // ConnectNodesStarPivot connects all nodes in a star topology // with the center at already set pivot node. // If ids argument is nil, all nodes that are up will be connected. -func (s *Simulation) ConnectNodesStarPivot(ids []enode.ID) (err error) { - id := s.PivotNodeID() - if id == nil { +func (net *Network) ConnectNodesStarPivot(ids []enode.ID) (err error) { + pivot := net.GetPivotNode() + if pivot == nil { return ErrNoPivotNode } - return s.ConnectNodesStar(*id, ids) + return net.ConnectNodesStar(pivot.ID(), ids) } // connect connects two nodes but ignores already connected error. -func (s *Simulation) connect(oneID, otherID enode.ID) error { - return ignoreAlreadyConnectedErr(s.Net.Connect(oneID, otherID)) +func (net *Network) connect(oneID, otherID enode.ID) error { + return ignoreAlreadyConnectedErr(net.Connect(oneID, otherID)) } func ignoreAlreadyConnectedErr(err error) error { @@ -157,3 +155,22 @@ func ignoreAlreadyConnectedErr(err error) error { } return err } + +// SetPivotNode sets the NodeID of the network's pivot node. +// Pivot node is just a specific node that should be treated +// differently then other nodes in test. SetPivotNode and +// GetPivotNode are just a convenient functions to set and +// retrieve it. +func (net *Network) SetPivotNode(id enode.ID) { + net.lock.Lock() + defer net.lock.Unlock() + net.pivotNodeID = id +} + +// GetPivotNode returns NodeID of the pivot node set by +// Network.SetPivotNode method. +func (net *Network) GetPivotNode() (node *Node) { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getNode(net.pivotNodeID) +} diff --git a/p2p/simulations/connect_test.go b/p2p/simulations/connect_test.go new file mode 100644 index 0000000000..36f9442a8a --- /dev/null +++ b/p2p/simulations/connect_test.go @@ -0,0 +1,190 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package simulations + +import ( + "testing" + + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" +) + +func newTestNetwork(t *testing.T, nodeCount int) (*Network, []enode.ID) { + adapter := adapters.NewSimAdapter(adapters.Services{ + "noopwoop": func(ctx *adapters.ServiceContext) (node.Service, error) { + return NewNoopService(nil), nil + }, + }) + + // create network + network := NewNetwork(adapter, &NetworkConfig{ + DefaultService: "noopwoop", + }) + + // create and start nodes + ids := make([]enode.ID, nodeCount) + for i := range ids { + conf := adapters.RandomNodeConfig() + node, err := network.NewNodeWithConfig(conf) + if err != nil { + t.Fatalf("error creating node: %s", err) + } + if err := network.Start(node.ID()); err != nil { + t.Fatalf("error starting node: %s", err) + } + ids[i] = node.ID() + } + + if len(network.Conns) > 0 { + t.Fatal("no connections should exist after just adding nodes") + } + + return network, ids +} + +func TestConnectToPivotNode(t *testing.T) { + net, ids := newTestNetwork(t, 2) + defer net.Shutdown() + + pivot := ids[0] + net.SetPivotNode(pivot) + + other := ids[1] + err := net.ConnectToPivotNode(other) + if err != nil { + t.Fatal(err) + } + + if net.GetConn(pivot, other) == nil { + t.Error("pivot and the other node are not connected") + } +} + +func TestConnectToLastNode(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + first := ids[0] + if err := net.ConnectToLastNode(first); err != nil { + t.Fatal(err) + } + + last := ids[len(ids)-1] + for i, id := range ids { + if id == first || id == last { + continue + } + + if net.GetConn(first, id) != nil { + t.Errorf("connection must not exist with node(ind: %v, id: %v)", i, id) + } + } + + if net.GetConn(first, last) == nil { + t.Error("first and last node must be connected") + } +} + +func TestConnectToRandomNode(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectToRandomNode(ids[0]) + if err != nil { + t.Fatal(err) + } + + var cc int + for i, a := range ids { + for _, b := range ids[i:] { + if net.GetConn(a, b) != nil { + cc++ + } + } + } + + if cc != 1 { + t.Errorf("expected one connection, got %v", cc) + } +} + +func TestConnectNodesFull(t *testing.T) { + net, ids := newTestNetwork(t, 12) + defer net.Shutdown() + + err := net.ConnectNodesFull(ids) + if err != nil { + t.Fatal(err) + } + + VerifyFull(t, net, ids) +} + +func TestConnectNodesChain(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectNodesChain(ids) + if err != nil { + t.Fatal(err) + } + + VerifyChain(t, net, ids) +} + +func TestConnectNodesRing(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + err := net.ConnectNodesRing(ids) + if err != nil { + t.Fatal(err) + } + + VerifyRing(t, net, ids) +} + +func TestConnectNodesStar(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + pivotIndex := 2 + + err := net.ConnectNodesStar(ids[pivotIndex], ids) + if err != nil { + t.Fatal(err) + } + + VerifyStar(t, net, ids, pivotIndex) +} + +func TestConnectNodesStarPivot(t *testing.T) { + net, ids := newTestNetwork(t, 10) + defer net.Shutdown() + + pivotIndex := 4 + + net.SetPivotNode(ids[pivotIndex]) + + err := net.ConnectNodesStarPivot(ids) + if err != nil { + t.Fatal(err) + } + + VerifyStar(t, net, ids, pivotIndex) +} diff --git a/p2p/simulations/network.go b/p2p/simulations/network.go index ab9f582c52..a6fac2c2af 100644 --- a/p2p/simulations/network.go +++ b/p2p/simulations/network.go @@ -22,6 +22,7 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "sync" "time" @@ -57,6 +58,8 @@ type Network struct { Conns []*Conn `json:"conns"` connMap map[string]int + pivotNodeID enode.ID + nodeAdapter adapters.NodeAdapter events event.Feed lock sync.RWMutex @@ -370,23 +373,32 @@ func (net *Network) DidReceive(sender, receiver enode.ID, proto string, code uin // GetNode gets the node with the given ID, returning nil if the node does not // exist func (net *Network) GetNode(id enode.ID) *Node { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getNode(id) } // GetNode gets the node with the given name, returning nil if the node does // not exist func (net *Network) GetNodeByName(name string) *Node { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getNodeByName(name) } +func (net *Network) getNodeByName(name string) *Node { + for _, node := range net.Nodes { + if node.Config.Name == name { + return node + } + } + return nil +} + // GetNodes returns the existing nodes func (net *Network) GetNodes() (nodes []*Node) { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() nodes = append(nodes, net.Nodes...) return nodes @@ -400,20 +412,67 @@ func (net *Network) getNode(id enode.ID) *Node { return net.Nodes[i] } -func (net *Network) getNodeByName(name string) *Node { +// GetRandomUpNode returns a random node on the network, which is running. +func (net *Network) GetRandomUpNode(excludeIDs ...enode.ID) *Node { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getRandomNode(net.getUpNodeIDs(), excludeIDs) +} + +func (net *Network) getUpNodeIDs() (ids []enode.ID) { for _, node := range net.Nodes { - if node.Config.Name == name { - return node + if node.Up { + ids = append(ids, node.ID()) } } - return nil + return ids +} + +// GetRandomDownNode returns a random node on the network, which is stopped. +func (net *Network) GetRandomDownNode(excludeIDs ...enode.ID) *Node { + net.lock.RLock() + defer net.lock.RUnlock() + return net.getRandomNode(net.getDownNodeIDs(), excludeIDs) +} + +func (net *Network) getDownNodeIDs() (ids []enode.ID) { + for _, node := range net.GetNodes() { + if !node.Up { + ids = append(ids, node.ID()) + } + } + return ids +} + +func (net *Network) getRandomNode(ids []enode.ID, excludeIDs []enode.ID) *Node { + filtered := filterIDs(ids, excludeIDs) + + l := len(filtered) + if l == 0 { + return nil + } + return net.GetNode(filtered[rand.Intn(l)]) +} + +func filterIDs(ids []enode.ID, excludeIDs []enode.ID) []enode.ID { + exclude := make(map[enode.ID]bool) + for _, id := range excludeIDs { + exclude[id] = true + } + var filtered []enode.ID + for _, id := range ids { + if _, found := exclude[id]; !found { + filtered = append(filtered, id) + } + } + return filtered } // GetConn returns the connection which exists between "one" and "other" // regardless of which node initiated the connection func (net *Network) GetConn(oneID, otherID enode.ID) *Conn { - net.lock.Lock() - defer net.lock.Unlock() + net.lock.RLock() + defer net.lock.RUnlock() return net.getConn(oneID, otherID) } diff --git a/p2p/simulations/test.go b/p2p/simulations/test.go new file mode 100644 index 0000000000..beeb414e41 --- /dev/null +++ b/p2p/simulations/test.go @@ -0,0 +1,134 @@ +package simulations + +import ( + "testing" + + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/rpc" +) + +// NoopService is the service that does not do anything +// but implements node.Service interface. +type NoopService struct { + c map[enode.ID]chan struct{} +} + +func NewNoopService(ackC map[enode.ID]chan struct{}) *NoopService { + return &NoopService{ + c: ackC, + } +} + +func (t *NoopService) Protocols() []p2p.Protocol { + return []p2p.Protocol{ + { + Name: "noop", + Version: 666, + Length: 0, + Run: func(peer *p2p.Peer, rw p2p.MsgReadWriter) error { + if t.c != nil { + t.c[peer.ID()] = make(chan struct{}) + close(t.c[peer.ID()]) + } + rw.ReadMsg() + return nil + }, + NodeInfo: func() interface{} { + return struct{}{} + }, + PeerInfo: func(id enode.ID) interface{} { + return struct{}{} + }, + Attributes: []enr.Entry{}, + }, + } +} + +func (t *NoopService) APIs() []rpc.API { + return []rpc.API{} +} + +func (t *NoopService) Start(server *p2p.Server) error { + return nil +} + +func (t *NoopService) Stop() error { + return nil +} + +func VerifyRing(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == j-1 || (i == 0 && j == n-1) { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} + +func VerifyChain(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == j-1 { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} + +func VerifyFull(t *testing.T, net *Network, ids []enode.ID) { + t.Helper() + n := len(ids) + var connections int + for i, lid := range ids { + for _, rid := range ids[i+1:] { + if net.GetConn(lid, rid) != nil { + connections++ + } + } + } + + want := n * (n - 1) / 2 + if connections != want { + t.Errorf("wrong number of connections, got: %v, want: %v", connections, want) + } +} + +func VerifyStar(t *testing.T, net *Network, ids []enode.ID, centerIndex int) { + t.Helper() + n := len(ids) + for i := 0; i < n; i++ { + for j := i + 1; j < n; j++ { + c := net.GetConn(ids[i], ids[j]) + if i == centerIndex || j == centerIndex { + if c == nil { + t.Errorf("nodes %v and %v are not connected, but they should be", i, j) + } + } else { + if c != nil { + t.Errorf("nodes %v and %v are connected, but they should not be", i, j) + } + } + } + } +} diff --git a/swarm/network/simulation/connect_test.go b/swarm/network/simulation/connect_test.go deleted file mode 100644 index 6c94b3a01c..0000000000 --- a/swarm/network/simulation/connect_test.go +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package simulation - -import ( - "testing" - - "github.com/ethereum/go-ethereum/p2p/enode" -) - -func TestConnectToPivotNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - pid, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - sim.SetPivotNode(pid) - - id, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToPivotNode(id) - if err != nil { - t.Fatal(err) - } - - if sim.Net.GetConn(id, pid) == nil { - t.Error("node did not connect to pivot node") - } -} - -func TestConnectToLastNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - n := 10 - - ids, err := sim.AddNodes(n) - if err != nil { - t.Fatal(err) - } - - id, err := sim.AddNode() - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToLastNode(id) - if err != nil { - t.Fatal(err) - } - - for _, i := range ids[:n-2] { - if sim.Net.GetConn(id, i) != nil { - t.Error("node connected to the node that is not the last") - } - } - - if sim.Net.GetConn(id, ids[n-1]) == nil { - t.Error("node did not connect to the last node") - } -} - -func TestConnectToRandomNode(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - n := 10 - - ids, err := sim.AddNodes(n) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectToRandomNode(ids[0]) - if err != nil { - t.Fatal(err) - } - - var cc int - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - if sim.Net.GetConn(ids[i], ids[j]) != nil { - cc++ - } - } - } - - if cc != 1 { - t.Errorf("expected one connection, got %v", cc) - } -} - -func TestConnectNodesFull(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(12) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesFull(ids) - if err != nil { - t.Fatal(err) - } - - testFull(t, sim, ids) -} - -func testFull(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - var cc int - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - if sim.Net.GetConn(ids[i], ids[j]) != nil { - cc++ - } - } - } - - want := n * (n - 1) / 2 - - if cc != want { - t.Errorf("expected %v connection, got %v", want, cc) - } -} - -func TestConnectNodesChain(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesChain(ids) - if err != nil { - t.Fatal(err) - } - - testChain(t, sim, ids) -} - -func testChain(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == j-1 { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectNodesRing(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - err = sim.ConnectNodesRing(ids) - if err != nil { - t.Fatal(err) - } - - testRing(t, sim, ids) -} - -func testRing(t *testing.T, sim *Simulation, ids []enode.ID) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == j-1 || (i == 0 && j == n-1) { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectToNodesStar(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - centerIndex := 2 - - err = sim.ConnectNodesStar(ids[centerIndex], ids) - if err != nil { - t.Fatal(err) - } - - testStar(t, sim, ids, centerIndex) -} - -func testStar(t *testing.T, sim *Simulation, ids []enode.ID, centerIndex int) { - n := len(ids) - for i := 0; i < n; i++ { - for j := i + 1; j < n; j++ { - c := sim.Net.GetConn(ids[i], ids[j]) - if i == centerIndex || j == centerIndex { - if c == nil { - t.Errorf("nodes %v and %v are not connected, but they should be", i, j) - } - } else { - if c != nil { - t.Errorf("nodes %v and %v are connected, but they should not be", i, j) - } - } - } - } -} - -func TestConnectToNodesStarPivot(t *testing.T) { - sim := New(noopServiceFuncMap) - defer sim.Close() - - ids, err := sim.AddNodes(10) - if err != nil { - t.Fatal(err) - } - - if len(sim.Net.Conns) > 0 { - t.Fatal("no connections should exist after just adding nodes") - } - - pivotIndex := 4 - - sim.SetPivotNode(ids[pivotIndex]) - - err = sim.ConnectNodesStarPivot(ids) - if err != nil { - t.Fatal(err) - } - - testStar(t, sim, ids, pivotIndex) -} diff --git a/swarm/network/simulation/events_test.go b/swarm/network/simulation/events_test.go index 0c185d9774..34ef24ed40 100644 --- a/swarm/network/simulation/events_test.go +++ b/swarm/network/simulation/events_test.go @@ -59,7 +59,7 @@ func TestPeerEvents(t *testing.T) { } }() - err = sim.ConnectNodesChain(sim.NodeIDs()) + err = sim.Net.ConnectNodesChain(sim.NodeIDs()) if err != nil { t.Fatal(err) } diff --git a/swarm/network/simulation/node.go b/swarm/network/simulation/node.go index a916d3fc21..24b6599762 100644 --- a/swarm/network/simulation/node.go +++ b/swarm/network/simulation/node.go @@ -127,7 +127,7 @@ func (s *Simulation) AddNodesAndConnectFull(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesFull(ids) + err = s.Net.ConnectNodesFull(ids) if err != nil { return nil, err } @@ -145,7 +145,7 @@ func (s *Simulation) AddNodesAndConnectChain(count int, opts ...AddNodeOption) ( if err != nil { return nil, err } - err = s.ConnectToLastNode(id) + err = s.Net.ConnectToLastNode(id) if err != nil { return nil, err } @@ -154,7 +154,7 @@ func (s *Simulation) AddNodesAndConnectChain(count int, opts ...AddNodeOption) ( return nil, err } ids = append([]enode.ID{id}, ids...) - err = s.ConnectNodesChain(ids) + err = s.Net.ConnectNodesChain(ids) if err != nil { return nil, err } @@ -171,7 +171,7 @@ func (s *Simulation) AddNodesAndConnectRing(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesRing(ids) + err = s.Net.ConnectNodesRing(ids) if err != nil { return nil, err } @@ -188,7 +188,7 @@ func (s *Simulation) AddNodesAndConnectStar(count int, opts ...AddNodeOption) (i if err != nil { return nil, err } - err = s.ConnectNodesStar(ids[0], ids[1:]) + err = s.Net.ConnectNodesStar(ids[0], ids[1:]) if err != nil { return nil, err } @@ -267,27 +267,26 @@ func (s *Simulation) StartNode(id enode.ID) (err error) { // StartRandomNode starts a random node. func (s *Simulation) StartRandomNode() (id enode.ID, err error) { - n := s.randomDownNode() + n := s.Net.GetRandomDownNode() if n == nil { return id, ErrNodeNotFound } - return n.ID, s.Net.Start(n.ID) + return n.ID(), s.Net.Start(n.ID()) } // StartRandomNodes starts random nodes. func (s *Simulation) StartRandomNodes(count int) (ids []enode.ID, err error) { ids = make([]enode.ID, 0, count) - downIDs := s.DownNodeIDs() for i := 0; i < count; i++ { - n := s.randomNode(downIDs, ids...) + n := s.Net.GetRandomDownNode() if n == nil { return nil, ErrNodeNotFound } - err = s.Net.Start(n.ID) + err = s.Net.Start(n.ID()) if err != nil { return nil, err } - ids = append(ids, n.ID) + ids = append(ids, n.ID()) } return ids, nil } @@ -299,27 +298,26 @@ func (s *Simulation) StopNode(id enode.ID) (err error) { // StopRandomNode stops a random node. func (s *Simulation) StopRandomNode() (id enode.ID, err error) { - n := s.RandomUpNode() + n := s.Net.GetRandomUpNode() if n == nil { return id, ErrNodeNotFound } - return n.ID, s.Net.Stop(n.ID) + return n.ID(), s.Net.Stop(n.ID()) } // StopRandomNodes stops random nodes. func (s *Simulation) StopRandomNodes(count int) (ids []enode.ID, err error) { ids = make([]enode.ID, 0, count) - upIDs := s.UpNodeIDs() for i := 0; i < count; i++ { - n := s.randomNode(upIDs, ids...) + n := s.Net.GetRandomUpNode() if n == nil { return nil, ErrNodeNotFound } - err = s.Net.Stop(n.ID) + err = s.Net.Stop(n.ID()) if err != nil { return nil, err } - ids = append(ids, n.ID) + ids = append(ids, n.ID()) } return ids, nil } @@ -328,35 +326,3 @@ func (s *Simulation) StopRandomNodes(count int) (ids []enode.ID, err error) { func init() { rand.Seed(time.Now().UnixNano()) } - -// RandomUpNode returns a random SimNode that is up. -// Arguments are NodeIDs for nodes that should not be returned. -func (s *Simulation) RandomUpNode(exclude ...enode.ID) *adapters.SimNode { - return s.randomNode(s.UpNodeIDs(), exclude...) -} - -// randomDownNode returns a random SimNode that is not up. -func (s *Simulation) randomDownNode(exclude ...enode.ID) *adapters.SimNode { - return s.randomNode(s.DownNodeIDs(), exclude...) -} - -// randomNode returns a random SimNode from the slice of NodeIDs. -func (s *Simulation) randomNode(ids []enode.ID, exclude ...enode.ID) *adapters.SimNode { - for _, e := range exclude { - var i int - for _, id := range ids { - if id == e { - ids = append(ids[:i], ids[i+1:]...) - } else { - i++ - } - } - } - l := len(ids) - if l == 0 { - return nil - } - n := s.Net.GetNode(ids[rand.Intn(l)]) - node, _ := n.Node.(*adapters.SimNode) - return node -} diff --git a/swarm/network/simulation/node_test.go b/swarm/network/simulation/node_test.go index 01346ef14d..8da32cf37f 100644 --- a/swarm/network/simulation/node_test.go +++ b/swarm/network/simulation/node_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/simulations" "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/swarm/network" ) @@ -228,7 +229,7 @@ func TestAddNodesAndConnectFull(t *testing.T) { t.Fatal(err) } - testFull(t, sim, ids) + simulations.VerifyFull(t, sim.Net, ids) } func TestAddNodesAndConnectChain(t *testing.T) { @@ -247,7 +248,7 @@ func TestAddNodesAndConnectChain(t *testing.T) { t.Fatal(err) } - testChain(t, sim, sim.UpNodeIDs()) + simulations.VerifyChain(t, sim.Net, sim.UpNodeIDs()) } func TestAddNodesAndConnectRing(t *testing.T) { @@ -259,7 +260,7 @@ func TestAddNodesAndConnectRing(t *testing.T) { t.Fatal(err) } - testRing(t, sim, ids) + simulations.VerifyRing(t, sim.Net, ids) } func TestAddNodesAndConnectStar(t *testing.T) { @@ -271,7 +272,7 @@ func TestAddNodesAndConnectStar(t *testing.T) { t.Fatal(err) } - testStar(t, sim, ids, 0) + simulations.VerifyStar(t, sim.Net, ids, 0) } //To test that uploading a snapshot works diff --git a/swarm/network/simulation/service.go b/swarm/network/simulation/service.go index 819602e9e8..7dd4dc6d82 100644 --- a/swarm/network/simulation/service.go +++ b/swarm/network/simulation/service.go @@ -39,7 +39,7 @@ func (s *Simulation) Service(name string, id enode.ID) node.Service { // RandomService returns a single Service by name on a // randomly chosen node that is up. func (s *Simulation) RandomService(name string) node.Service { - n := s.RandomUpNode() + n := s.Net.GetRandomUpNode().Node.(*adapters.SimNode) if n == nil { return nil } diff --git a/swarm/network/simulation/simulation.go b/swarm/network/simulation/simulation.go index e5435b9f0e..747faf5d77 100644 --- a/swarm/network/simulation/simulation.go +++ b/swarm/network/simulation/simulation.go @@ -33,7 +33,6 @@ import ( // Common errors that are returned by functions in this package. var ( ErrNodeNotFound = errors.New("node not found") - ErrNoPivotNode = errors.New("no pivot node set") ) // Simulation provides methods on network, nodes and services diff --git a/swarm/network/stream/delivery_test.go b/swarm/network/stream/delivery_test.go index f69f80499b..f537c13234 100644 --- a/swarm/network/stream/delivery_test.go +++ b/swarm/network/stream/delivery_test.go @@ -505,7 +505,8 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { nodeIDs := sim.UpNodeIDs() //determine the pivot node to be the first node of the simulation - sim.SetPivotNode(nodeIDs[0]) + pivot := nodeIDs[0] + //distribute chunks of a random file into Stores of nodes 1 to nodes //we will do this by creating a file store with an underlying round-robin store: //the file store will create a hash for the uploaded file, but every chunk will be @@ -519,7 +520,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck //...iterate the buckets... for id, bucketVal := range lStores { //...and remove the one which is the pivot node - if id == *sim.PivotNodeID() { + if id == pivot { continue } //the other ones are added to the array... @@ -547,7 +548,7 @@ func testDeliveryFromNodes(t *testing.T, nodes, conns, chunkCount int, skipCheck } //get the pivot node's filestore - item, ok := sim.NodeItem(*sim.PivotNodeID(), bucketKeyFileStore) + item, ok := sim.NodeItem(pivot, bucketKeyFileStore) if !ok { return fmt.Errorf("No filestore") } diff --git a/swarm/network/stream/snapshot_retrieval_test.go b/swarm/network/stream/snapshot_retrieval_test.go index 932e28b32f..62b53fad3b 100644 --- a/swarm/network/stream/snapshot_retrieval_test.go +++ b/swarm/network/stream/snapshot_retrieval_test.go @@ -278,13 +278,13 @@ func runRetrievalTest(chunkCount int, nodeCount int) error { } //this is the node selected for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - conf.hashes, err = uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + conf.hashes, err = uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } diff --git a/swarm/network/stream/snapshot_sync_test.go b/swarm/network/stream/snapshot_sync_test.go index 4a632c8c90..40b4327fd1 100644 --- a/swarm/network/stream/snapshot_sync_test.go +++ b/swarm/network/stream/snapshot_sync_test.go @@ -248,20 +248,20 @@ func runSim(conf *synctestConfig, ctx context.Context, sim *simulation.Simulatio //get the node at that index //this is the node selected for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - hashes, err := uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + hashes, err := uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } for _, h := range hashes { evt := &simulations.Event{ Type: EventTypeChunkCreated, - Node: sim.Net.GetNode(node.ID), + Node: sim.Net.GetNode(node.ID()), Data: h.String(), } sim.Net.Events().Send(evt) @@ -453,13 +453,13 @@ func testSyncingViaDirectSubscribe(t *testing.T, chunkCount int, nodeCount int) } } //select a random node for upload - node := sim.RandomUpNode() - item, ok := sim.NodeItem(node.ID, bucketKeyStore) + node := sim.Net.GetRandomUpNode() + item, ok := sim.NodeItem(node.ID(), bucketKeyStore) if !ok { return fmt.Errorf("No localstore") } lstore := item.(*storage.LocalStore) - hashes, err := uploadFileToSingleNodeStore(node.ID, chunkCount, lstore) + hashes, err := uploadFileToSingleNodeStore(node.ID(), chunkCount, lstore) if err != nil { return err } diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index f6d618020d..27237d1849 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -103,7 +103,7 @@ func TestNonExistingHashesWithServer(t *testing.T) { result := sim.Run(ctx, func(ctx context.Context, sim *simulation.Simulation) error { //check on the node's FileStore (netstore) - id := sim.RandomUpNode().ID + id := sim.Net.GetRandomUpNode().ID() item, ok := sim.NodeItem(id, bucketKeyFileStore) if !ok { t.Fatalf("No filestore") From 90ea542e9efa5ec7652e2b507a7122e5a8e4b91b Mon Sep 17 00:00:00 2001 From: holisticode Date: Tue, 18 Dec 2018 01:20:59 -0500 Subject: [PATCH 65/68] Update visualized snapshot test (#18286) * swarm/network/stream: fix visualized_snapshot_sync_sim_test * swarm/network/stream: updated visualized snapshot-test;data in p2p event * swarm/network/stream: cleanup visualized snapshot sync test * swarm/network/stream: re-enable t.Skip for visualized test * swarm/network/stream: addressed PR comments --- .../visualized_snapshot_sync_sim_test.go | 267 ++++++++++++++---- 1 file changed, 214 insertions(+), 53 deletions(-) diff --git a/swarm/network/stream/visualized_snapshot_sync_sim_test.go b/swarm/network/stream/visualized_snapshot_sync_sim_test.go index 27237d1849..96b707797c 100644 --- a/swarm/network/stream/visualized_snapshot_sync_sim_test.go +++ b/swarm/network/stream/visualized_snapshot_sync_sim_test.go @@ -19,16 +19,27 @@ package stream import ( + "bytes" "context" + "errors" "fmt" + "io" + "os" + "sync" "testing" "time" + "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/discover" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/protocols" "github.com/ethereum/go-ethereum/p2p/simulations" + "github.com/ethereum/go-ethereum/p2p/simulations/adapters" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/swarm/log" + "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" + "github.com/ethereum/go-ethereum/swarm/state" "github.com/ethereum/go-ethereum/swarm/storage" ) @@ -68,12 +79,12 @@ func watchSim(sim *simulation.Simulation) (context.Context, context.CancelFunc) disconnections := sim.PeerEvents( context.Background(), sim.NodeIDs(), - simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeDrop), + simulation.NewPeerEventsFilter().Drop(), ) go func() { for d := range disconnections { - log.Error("peer drop", "node", d.NodeID, "peer", d.Event.Peer) + log.Error("peer drop", "node", d.NodeID, "peer", d.PeerID) panic("unexpected disconnect") cancelSimRun() } @@ -144,21 +155,75 @@ func sendSimTerminatedEvent(sim *simulation.Simulation) { //It also sends some custom events so that the frontend //can visualize messages like SendOfferedMsg, WantedHashesMsg, DeliveryMsg func TestSnapshotSyncWithServer(t *testing.T) { + //t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") + + //define a wrapper object to be able to pass around data + wrapper := &netWrapper{} + + nodeCount := *nodes + chunkCount := *chunks + + if nodeCount == 0 || chunkCount == 0 { + nodeCount = 32 + chunkCount = 1 + } + + log.Info(fmt.Sprintf("Running the simulation with %d nodes and %d chunks", nodeCount, chunkCount)) + + sim := simulation.New(map[string]simulation.ServiceFunc{ + "streamer": func(ctx *adapters.ServiceContext, bucket *sync.Map) (s node.Service, cleanup func(), err error) { + n := ctx.Config.Node() + addr := network.NewAddr(n) + store, datadir, err := createTestLocalStorageForID(n.ID(), addr) + if err != nil { + return nil, nil, err + } + bucket.Store(bucketKeyStore, store) + localStore := store.(*storage.LocalStore) + netStore, err := storage.NewNetStore(localStore, nil) + if err != nil { + return nil, nil, err + } + kad := network.NewKademlia(addr.Over(), network.NewKadParams()) + delivery := NewDelivery(kad, netStore) + netStore.NewNetFetcherFunc = network.NewFetcherFactory(dummyRequestFromPeers, true).New + + r := NewRegistry(addr.ID(), delivery, netStore, state.NewInmemoryStore(), &RegistryOptions{ + Retrieval: RetrievalDisabled, + Syncing: SyncingAutoSubscribe, + SyncUpdateDelay: 3 * time.Second, + }, nil) + + tr := &testRegistry{ + Registry: r, + w: wrapper, + } + + bucket.Store(bucketKeyRegistry, tr) + + cleanup = func() { + netStore.Close() + tr.Close() + os.RemoveAll(datadir) + } + + return tr, cleanup, nil + }, + }).WithServer(":8888") //start with the HTTP server - t.Skip("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - nodeCount, chunkCount, sim := setupSim(simServiceMap) defer sim.Close() log.Info("Initializing test config") conf := &synctestConfig{} //map of discover ID to indexes of chunks expected at that ID - conf.idToChunksMap = make(map[discover.NodeID][]int) + conf.idToChunksMap = make(map[enode.ID][]int) //map of overlay address to discover ID - conf.addrToIDMap = make(map[string]discover.NodeID) + conf.addrToIDMap = make(map[string]enode.ID) //array where the generated chunk hashes will be stored conf.hashes = make([]storage.Address, 0) - + //pass the network to the wrapper object + wrapper.setNetwork(sim.Net) err := sim.UploadSnapshot(fmt.Sprintf("testing/snapshot_%d.json", nodeCount)) if err != nil { panic(err) @@ -167,49 +232,6 @@ func TestSnapshotSyncWithServer(t *testing.T) { ctx, cancelSimRun := watchSim(sim) defer cancelSimRun() - //setup filters in the event feed - offeredHashesFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(1) - wantedFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(2) - deliveryFilter := simulation.NewPeerEventsFilter().Type(p2p.PeerEventTypeMsgRecv).Protocol("stream").MsgCode(6) - eventC := sim.PeerEvents(ctx, sim.UpNodeIDs(), offeredHashesFilter, wantedFilter, deliveryFilter) - - quit := make(chan struct{}) - - go func() { - for e := range eventC { - select { - case <-quit: - fmt.Println("quitting event loop") - return - default: - } - if e.Error != nil { - t.Fatal(e.Error) - } - if *e.Event.MsgCode == uint64(1) { - evt := &simulations.Event{ - Type: EventTypeChunkOffered, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } else if *e.Event.MsgCode == uint64(2) { - evt := &simulations.Event{ - Type: EventTypeChunkWanted, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } else if *e.Event.MsgCode == uint64(6) { - evt := &simulations.Event{ - Type: EventTypeChunkDelivered, - Node: sim.Net.GetNode(e.NodeID), - Control: false, - } - sim.Net.Events().Send(evt) - } - } - }() //run the sim result := runSim(conf, ctx, sim, chunkCount) @@ -218,11 +240,150 @@ func TestSnapshotSyncWithServer(t *testing.T) { Type: EventTypeSimTerminated, Control: false, } - sim.Net.Events().Send(evt) + go sim.Net.Events().Send(evt) if result.Error != nil { panic(result.Error) } - close(quit) log.Info("Simulation ended") } + +//testRegistry embeds registry +//it allows to replace the protocol run function +type testRegistry struct { + *Registry + w *netWrapper +} + +//Protocols replaces the protocol's run function +func (tr *testRegistry) Protocols() []p2p.Protocol { + regProto := tr.Registry.Protocols() + //set the `stream` protocol's run function with the testRegistry's one + regProto[0].Run = tr.runProto + return regProto +} + +//runProto is the new overwritten protocol's run function for this test +func (tr *testRegistry) runProto(p *p2p.Peer, rw p2p.MsgReadWriter) error { + //create a custom rw message ReadWriter + testRw := &testMsgReadWriter{ + MsgReadWriter: rw, + Peer: p, + w: tr.w, + Registry: tr.Registry, + } + //now run the actual upper layer `Registry`'s protocol function + return tr.runProtocol(p, testRw) +} + +//testMsgReadWriter is a custom rw +//it will allow us to re-use the message twice +type testMsgReadWriter struct { + *Registry + p2p.MsgReadWriter + *p2p.Peer + w *netWrapper +} + +//netWrapper wrapper object so we can pass data around +type netWrapper struct { + net *simulations.Network +} + +//set the network to the wrapper for later use (used inside the custom rw) +func (w *netWrapper) setNetwork(n *simulations.Network) { + w.net = n +} + +//get he network from the wrapper (used inside the custom rw) +func (w *netWrapper) getNetwork() *simulations.Network { + return w.net +} + +// ReadMsg reads a message from the underlying MsgReadWriter and emits a +// "message received" event +//we do this because we are interested in the Payload of the message for custom use +//in this test, but messages can only be consumed once (stream io.Reader) +func (ev *testMsgReadWriter) ReadMsg() (p2p.Msg, error) { + //read the message from the underlying rw + msg, err := ev.MsgReadWriter.ReadMsg() + if err != nil { + return msg, err + } + + //don't do anything with message codes we actually are not needing/reading + subCodes := []uint64{1, 2, 10} + found := false + for _, c := range subCodes { + if c == msg.Code { + found = true + } + } + //just return if not a msg code we are interested in + if !found { + return msg, nil + } + + //we use a io.TeeReader so that we can read the message twice + //the Payload is a io.Reader, so if we read from it, the actual protocol handler + //cannot access it anymore. + //But we need that handler to be able to consume the message as normal, + //as if we would not do anything here with that message + var buf bytes.Buffer + tee := io.TeeReader(msg.Payload, &buf) + + mcp := &p2p.Msg{ + Code: msg.Code, + Size: msg.Size, + ReceivedAt: msg.ReceivedAt, + Payload: tee, + } + //assign the copy for later use + msg.Payload = &buf + + //now let's look into the message + var wmsg protocols.WrappedMsg + err = mcp.Decode(&wmsg) + if err != nil { + log.Error(err.Error()) + return msg, err + } + //create a new message from the code + val, ok := ev.Registry.GetSpec().NewMsg(mcp.Code) + if !ok { + return msg, errors.New(fmt.Sprintf("Invalid message code: %v", msg.Code)) + } + //decode it + if err := rlp.DecodeBytes(wmsg.Payload, val); err != nil { + return msg, errors.New(fmt.Sprintf("Decoding error <= %v: %v", msg, err)) + } + //now for every message type we are interested in, create a custom event and send it + var evt *simulations.Event + switch val := val.(type) { + case *OfferedHashesMsg: + evt = &simulations.Event{ + Type: EventTypeChunkOffered, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + Data: val.Hashes, + } + case *WantedHashesMsg: + evt = &simulations.Event{ + Type: EventTypeChunkWanted, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + } + case *ChunkDeliveryMsgSyncing: + evt = &simulations.Event{ + Type: EventTypeChunkDelivered, + Node: ev.w.getNetwork().GetNode(ev.ID()), + Control: false, + Data: val.Addr.String(), + } + } + if evt != nil { + //send custom event to feed; frontend will listen to it and display + ev.w.getNetwork().Events().Send(evt) + } + return msg, nil +} From de4265fa028f2b0eb96a21e295028e0d19426adb Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 18 Dec 2018 07:24:59 +0100 Subject: [PATCH 66/68] swarm/network/simulation:commented out unreachable code-avoid vet errors (#18263) --- swarm/network/simulation/example_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/swarm/network/simulation/example_test.go b/swarm/network/simulation/example_test.go index 7b62046174..a100ede516 100644 --- a/swarm/network/simulation/example_test.go +++ b/swarm/network/simulation/example_test.go @@ -18,14 +18,8 @@ package simulation_test import ( "context" - "fmt" - "sync" - "time" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/p2p/simulations/adapters" - "github.com/ethereum/go-ethereum/swarm/network" "github.com/ethereum/go-ethereum/swarm/network/simulation" ) @@ -35,7 +29,8 @@ import ( func ExampleSimulation_WaitTillHealthy() { log.Error("temporarily disabled as simulations.WaitTillHealthy cannot be trusted") - return + + /* Commented out to avoid go vet errors/warnings sim := simulation.New(map[string]simulation.ServiceFunc{ "bzz": func(ctx *adapters.ServiceContext, b *sync.Map) (node.Service, func(), error) { @@ -75,6 +70,8 @@ func ExampleSimulation_WaitTillHealthy() { } // continue with the test + + */ } // Watch all peer events in the simulation network, buy receiving from a channel. From b01cfce36276379a95deb1001a6f1b6a048609de Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 18 Dec 2018 15:23:32 +0100 Subject: [PATCH 67/68] swarm/pss: Reduce input vulnerabilities (#18304) --- swarm/pss/api.go | 26 ++++++++++--- swarm/pss/handshake.go | 8 ++-- swarm/pss/handshake_test.go | 1 + swarm/pss/notify/notify.go | 8 ++-- swarm/pss/pss.go | 74 +++++++++++++++++++++---------------- swarm/pss/pss_test.go | 59 +++++++++++++++++++++-------- 6 files changed, 116 insertions(+), 60 deletions(-) diff --git a/swarm/pss/api.go b/swarm/pss/api.go index 587382d729..4556d7b7c4 100644 --- a/swarm/pss/api.go +++ b/swarm/pss/api.go @@ -92,7 +92,7 @@ func (pssapi *API) Receive(ctx context.Context, topic Topic, raw bool, prox bool } func (pssapi *API) GetAddress(topic Topic, asymmetric bool, key string) (PssAddress, error) { - var addr *PssAddress + var addr PssAddress if asymmetric { peer, ok := pssapi.Pss.pubKeyPool[key][topic] if !ok { @@ -107,7 +107,7 @@ func (pssapi *API) GetAddress(topic Topic, asymmetric bool, key string) (PssAddr addr = peer.address } - return *addr, nil + return addr, nil } // Retrieves the node's base address in hex form @@ -128,7 +128,7 @@ func (pssapi *API) SetPeerPublicKey(pubkey hexutil.Bytes, topic Topic, addr PssA if err != nil { return fmt.Errorf("Cannot unmarshal pubkey: %x", pubkey) } - err = pssapi.Pss.SetPeerPublicKey(pk, topic, &addr) + err = pssapi.Pss.SetPeerPublicKey(pk, topic, addr) if err != nil { return fmt.Errorf("Invalid key: %x", pk) } @@ -141,11 +141,11 @@ func (pssapi *API) GetSymmetricKey(symkeyid string) (hexutil.Bytes, error) { } func (pssapi *API) GetSymmetricAddressHint(topic Topic, symkeyid string) (PssAddress, error) { - return *pssapi.Pss.symKeyPool[symkeyid][topic].address, nil + return pssapi.Pss.symKeyPool[symkeyid][topic].address, nil } func (pssapi *API) GetAsymmetricAddressHint(topic Topic, pubkeyid string) (PssAddress, error) { - return *pssapi.Pss.pubKeyPool[pubkeyid][topic].address, nil + return pssapi.Pss.pubKeyPool[pubkeyid][topic].address, nil } func (pssapi *API) StringToTopic(topicstring string) (Topic, error) { @@ -157,14 +157,23 @@ func (pssapi *API) StringToTopic(topicstring string) (Topic, error) { } func (pssapi *API) SendAsym(pubkeyhex string, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendAsym(pubkeyhex, topic, msg[:]) } func (pssapi *API) SendSym(symkeyhex string, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendSym(symkeyhex, topic, msg[:]) } func (pssapi *API) SendRaw(addr hexutil.Bytes, topic Topic, msg hexutil.Bytes) error { + if err := validateMsg(msg); err != nil { + return err + } return pssapi.Pss.SendRaw(PssAddress(addr), topic, msg[:]) } @@ -177,3 +186,10 @@ func (pssapi *API) GetPeerTopics(pubkeyhex string) ([]Topic, error) { func (pssapi *API) GetPeerAddress(pubkeyhex string, topic Topic) (PssAddress, error) { return pssapi.Pss.getPeerAddress(pubkeyhex, topic) } + +func validateMsg(msg []byte) error { + if len(msg) == 0 { + return errors.New("invalid message length") + } + return nil +} diff --git a/swarm/pss/handshake.go b/swarm/pss/handshake.go index 5486abafa9..bb67b51563 100644 --- a/swarm/pss/handshake.go +++ b/swarm/pss/handshake.go @@ -321,9 +321,7 @@ func (ctl *HandshakeController) handleKeys(pubkeyid string, keymsg *handshakeMsg for _, key := range keymsg.Keys { sendsymkey := make([]byte, len(key)) copy(sendsymkey, key) - var address PssAddress - copy(address[:], keymsg.From) - sendsymkeyid, err := ctl.pss.setSymmetricKey(sendsymkey, keymsg.Topic, &address, false, false) + sendsymkeyid, err := ctl.pss.setSymmetricKey(sendsymkey, keymsg.Topic, PssAddress(keymsg.From), false, false) if err != nil { return err } @@ -356,7 +354,7 @@ func (ctl *HandshakeController) handleKeys(pubkeyid string, keymsg *handshakeMsg func (ctl *HandshakeController) sendKey(pubkeyid string, topic *Topic, keycount uint8) ([]string, error) { var requestcount uint8 - to := &PssAddress{} + to := PssAddress{} if _, ok := ctl.pss.pubKeyPool[pubkeyid]; !ok { return []string{}, errors.New("Invalid public key") } else if psp, ok := ctl.pss.pubKeyPool[pubkeyid][*topic]; ok { @@ -564,5 +562,5 @@ func (api *HandshakeAPI) SendSym(symkeyid string, topic Topic, msg hexutil.Bytes api.ctrl.symKeyIndex[symkeyid].count++ log.Trace("increment symkey send use", "symkeyid", symkeyid, "count", api.ctrl.symKeyIndex[symkeyid].count, "limit", api.ctrl.symKeyIndex[symkeyid].limit, "receiver", common.ToHex(crypto.FromECDSAPub(api.ctrl.pss.PublicKey()))) } - return + return err } diff --git a/swarm/pss/handshake_test.go b/swarm/pss/handshake_test.go index 0fc7e798ff..895163f301 100644 --- a/swarm/pss/handshake_test.go +++ b/swarm/pss/handshake_test.go @@ -30,6 +30,7 @@ import ( // asymmetrical key exchange between two directly connected peers // full address, partial address (8 bytes) and empty address func TestHandshake(t *testing.T) { + t.Skip("handshakes are not adapted to current pss core code") t.Run("32", testHandshake) t.Run("8", testHandshake) t.Run("0", testHandshake) diff --git a/swarm/pss/notify/notify.go b/swarm/pss/notify/notify.go index d3c89058b2..e9d40dc321 100644 --- a/swarm/pss/notify/notify.go +++ b/swarm/pss/notify/notify.go @@ -138,7 +138,7 @@ func (c *Controller) Subscribe(name string, pubkey *ecdsa.PublicKey, address pss c.mu.Lock() defer c.mu.Unlock() msg := NewMsg(MsgCodeStart, name, c.pss.BaseAddr()) - c.pss.SetPeerPublicKey(pubkey, controlTopic, &address) + c.pss.SetPeerPublicKey(pubkey, controlTopic, address) pubkeyId := hexutil.Encode(crypto.FromECDSAPub(pubkey)) smsg, err := rlp.EncodeToBytes(msg) if err != nil { @@ -271,7 +271,7 @@ func (c *Controller) addToBin(ntfr *notifier, address []byte) (symKeyId string, currentBin.count++ symKeyId = currentBin.symKeyId } else { - symKeyId, err = c.pss.GenerateSymmetricKey(ntfr.topic, &pssAddress, false) + symKeyId, err = c.pss.GenerateSymmetricKey(ntfr.topic, pssAddress, false) if err != nil { return "", nil, err } @@ -312,7 +312,7 @@ func (c *Controller) handleStartMsg(msg *Msg, keyid string) (err error) { if err != nil { return err } - err = c.pss.SetPeerPublicKey(pubkey, controlTopic, &pssAddress) + err = c.pss.SetPeerPublicKey(pubkey, controlTopic, pssAddress) if err != nil { return err } @@ -335,7 +335,7 @@ func (c *Controller) handleNotifyWithKeyMsg(msg *Msg) error { // \TODO keep track of and add actual address updaterAddr := pss.PssAddress([]byte{}) - c.pss.SetSymmetricKey(symkey, topic, &updaterAddr, true) + c.pss.SetSymmetricKey(symkey, topic, updaterAddr, true) c.pss.Register(&topic, pss.NewHandler(c.Handler)) return c.subscriptions[msg.namestring].handler(msg.namestring, msg.Payload[:len(msg.Payload)-symKeyLength]) } diff --git a/swarm/pss/pss.go b/swarm/pss/pss.go index d0986d280b..1bc28890f9 100644 --- a/swarm/pss/pss.go +++ b/swarm/pss/pss.go @@ -81,7 +81,7 @@ type senderPeer interface { // member `protected` prevents garbage collection of the instance type pssPeer struct { lastSeen time.Time - address *PssAddress + address PssAddress protected bool } @@ -396,9 +396,11 @@ func (p *Pss) handlePssMsg(ctx context.Context, msg interface{}) error { // raw is simplest handler contingency to check, so check that first var isRaw bool if pssmsg.isRaw() { - if !p.topicHandlerCaps[psstopic].raw { - log.Debug("No handler for raw message", "topic", psstopic) - return nil + if _, ok := p.topicHandlerCaps[psstopic]; ok { + if !p.topicHandlerCaps[psstopic].raw { + log.Debug("No handler for raw message", "topic", psstopic) + return nil + } } isRaw = true } @@ -437,10 +439,10 @@ func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { var err error var recvmsg *whisper.ReceivedMessage var payload []byte - var from *PssAddress + var from PssAddress var asymmetric bool var keyid string - var keyFunc func(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) + var keyFunc func(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) envelope := pssmsg.Payload psstopic := Topic(envelope.Topic) @@ -473,7 +475,7 @@ func (p *Pss) process(pssmsg *PssMsg, raw bool, prox bool) error { } -func (p *Pss) executeHandlers(topic Topic, payload []byte, from *PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { +func (p *Pss) executeHandlers(topic Topic, payload []byte, from PssAddress, raw bool, prox bool, asymmetric bool, keyid string) { handlers := p.getHandlers(topic) peer := p2p.NewPeer(enode.ID{}, fmt.Sprintf("%x", from), []p2p.Cap{}) for h := range handlers { @@ -528,7 +530,10 @@ func (p *Pss) isSelfPossibleRecipient(msg *PssMsg, prox bool) bool { // // The value in `address` will be used as a routing hint for the // public key / topic association -func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address *PssAddress) error { +func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address PssAddress) error { + if err := validateAddress(address); err != nil { + return err + } pubkeybytes := crypto.FromECDSAPub(pubkey) if len(pubkeybytes) == 0 { return fmt.Errorf("invalid public key: %v", pubkey) @@ -543,12 +548,12 @@ func (p *Pss) SetPeerPublicKey(pubkey *ecdsa.PublicKey, topic Topic, address *Ps } p.pubKeyPool[pubkeyid][topic] = psp p.pubKeyPoolMu.Unlock() - log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", common.ToHex(*address)) + log.Trace("added pubkey", "pubkeyid", pubkeyid, "topic", topic, "address", address) return nil } // Automatically generate a new symkey for a topic and address hint -func (p *Pss) GenerateSymmetricKey(topic Topic, address *PssAddress, addToCache bool) (string, error) { +func (p *Pss) GenerateSymmetricKey(topic Topic, address PssAddress, addToCache bool) (string, error) { keyid, err := p.w.GenerateSymKey() if err != nil { return "", err @@ -569,11 +574,14 @@ func (p *Pss) GenerateSymmetricKey(topic Topic, address *PssAddress, addToCache // // Returns a string id that can be used to retrieve the key bytes // from the whisper backend (see pss.GetSymmetricKey()) -func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address *PssAddress, addtocache bool) (string, error) { +func (p *Pss) SetSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool) (string, error) { + if err := validateAddress(address); err != nil { + return "", err + } return p.setSymmetricKey(key, topic, address, addtocache, true) } -func (p *Pss) setSymmetricKey(key []byte, topic Topic, address *PssAddress, addtocache bool, protected bool) (string, error) { +func (p *Pss) setSymmetricKey(key []byte, topic Topic, address PssAddress, addtocache bool, protected bool) (string, error) { keyid, err := p.w.AddSymKeyDirect(key) if err != nil { return "", err @@ -585,7 +593,7 @@ func (p *Pss) setSymmetricKey(key []byte, topic Topic, address *PssAddress, addt // adds a symmetric key to the pss key pool, and optionally adds the key // to the collection of keys used to attempt symmetric decryption of // incoming messages -func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address *PssAddress, addtocache bool, protected bool) { +func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address PssAddress, addtocache bool, protected bool) { psp := &pssPeer{ address: address, protected: protected, @@ -601,7 +609,7 @@ func (p *Pss) addSymmetricKeyToPool(keyid string, topic Topic, address *PssAddre p.symKeyDecryptCache[p.symKeyDecryptCacheCursor%cap(p.symKeyDecryptCache)] = &keyid } key, _ := p.GetSymmetricKey(keyid) - log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", fmt.Sprintf("%p", address), "cache", addtocache) + log.Trace("added symkey", "symkeyid", keyid, "symkey", common.ToHex(key), "topic", topic, "address", address, "cache", addtocache) } // Returns a symmetric key byte seqyence stored in the whisper backend @@ -622,7 +630,7 @@ func (p *Pss) GetPublickeyPeers(keyid string) (topic []Topic, address []PssAddre defer p.pubKeyPoolMu.RUnlock() for t, peer := range p.pubKeyPool[keyid] { topic = append(topic, t) - address = append(address, *peer.address) + address = append(address, peer.address) } return topic, address, nil @@ -633,7 +641,7 @@ func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) { defer p.pubKeyPoolMu.RUnlock() if peers, ok := p.pubKeyPool[keyid]; ok { if t, ok := peers[topic]; ok { - return *t.address, nil + return t.address, nil } } return nil, fmt.Errorf("peer with pubkey %s, topic %x not found", keyid, topic) @@ -645,7 +653,7 @@ func (p *Pss) getPeerAddress(keyid string, topic Topic) (PssAddress, error) { // encapsulating the decrypted message, and the whisper backend id // of the symmetric key used to decrypt the message. // It fails if decryption of the message fails or if the message is corrupted -func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { +func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) { metrics.GetOrRegisterCounter("pss.process.sym", nil).Inc(1) for i := p.symKeyDecryptCacheCursor; i > p.symKeyDecryptCacheCursor-cap(p.symKeyDecryptCache) && i > 0; i-- { @@ -677,7 +685,7 @@ func (p *Pss) processSym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, // encapsulating the decrypted message, and the byte representation of // the public key used to decrypt the message. // It fails if decryption of message fails, or if the message is corrupted -func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, *PssAddress, error) { +func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, string, PssAddress, error) { metrics.GetOrRegisterCounter("pss.process.asym", nil).Inc(1) recvmsg, err := envelope.OpenAsymmetric(p.privateKey) @@ -689,7 +697,7 @@ func (p *Pss) processAsym(envelope *whisper.Envelope) (*whisper.ReceivedMessage, return nil, "", nil, fmt.Errorf("invalid message") } pubkeyid := common.ToHex(crypto.FromECDSAPub(recvmsg.Src)) - var from *PssAddress + var from PssAddress p.pubKeyPoolMu.Lock() if p.pubKeyPool[pubkeyid][Topic(envelope.Topic)] != nil { from = p.pubKeyPool[pubkeyid][Topic(envelope.Topic)].address @@ -751,6 +759,9 @@ func (p *Pss) enqueue(msg *PssMsg) error { // // Will fail if raw messages are disallowed func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { + if err := validateAddress(address); err != nil { + return err + } pssMsgParams := &msgParams{ raw: true, } @@ -770,8 +781,10 @@ func (p *Pss) SendRaw(address PssAddress, topic Topic, msg []byte) error { // if we have a proxhandler on this topic // also deliver message to ourselves - if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { - return p.process(pssMsg, true, true) + if _, ok := p.topicHandlerCaps[topic]; ok { + if p.isSelfPossibleRecipient(pssMsg, true) && p.topicHandlerCaps[topic].prox { + return p.process(pssMsg, true, true) + } } return nil } @@ -789,11 +802,8 @@ func (p *Pss) SendSym(symkeyid string, topic Topic, msg []byte) error { p.symKeyPoolMu.Unlock() if !ok { return fmt.Errorf("invalid topic '%s' for symkey '%s'", topic.String(), symkeyid) - } else if psp.address == nil { - return fmt.Errorf("no address hint for topic '%s' symkey '%s'", topic.String(), symkeyid) } - err = p.send(*psp.address, topic, msg, false, symkey) - return err + return p.send(psp.address, topic, msg, false, symkey) } // Send a message using asymmetric encryption @@ -808,13 +818,8 @@ func (p *Pss) SendAsym(pubkeyid string, topic Topic, msg []byte) error { p.pubKeyPoolMu.Unlock() if !ok { return fmt.Errorf("invalid topic '%s' for pubkey '%s'", topic.String(), pubkeyid) - } else if psp.address == nil { - return fmt.Errorf("no address hint for topic '%s' pubkey '%s'", topic.String(), pubkeyid) } - go func() { - p.send(*psp.address, topic, msg, true, common.FromHex(pubkeyid)) - }() - return nil + return p.send(psp.address, topic, msg, true, common.FromHex(pubkeyid)) } // Send is payload agnostic, and will accept any byte slice as payload @@ -1034,3 +1039,10 @@ func (p *Pss) digestBytes(msg []byte) pssDigest { copy(digest[:], key[:digestLength]) return digest } + +func validateAddress(addr PssAddress) error { + if len(addr) > addressLength { + return errors.New("address too long") + } + return nil +} diff --git a/swarm/pss/pss_test.go b/swarm/pss/pss_test.go index 72f62acd94..ec46504c23 100644 --- a/swarm/pss/pss_test.go +++ b/swarm/pss/pss_test.go @@ -407,7 +407,7 @@ func TestProxShortCircuit(t *testing.T) { // try the same prox message with sym and asym send proxAddrPss := PssAddress(proxMessageAddress) - symKeyId, err := ps.GenerateSymmetricKey(topic, &proxAddrPss, true) + symKeyId, err := ps.GenerateSymmetricKey(topic, proxAddrPss, true) go func() { err := ps.SendSym(symKeyId, topic, []byte("baz")) if err != nil { @@ -424,7 +424,7 @@ func TestProxShortCircuit(t *testing.T) { t.Fatal("sym timeout") } - err = ps.SetPeerPublicKey(&privKey.PublicKey, topic, &proxAddrPss) + err = ps.SetPeerPublicKey(&privKey.PublicKey, topic, proxAddrPss) if err != nil { t.Fatal(err) } @@ -786,14 +786,14 @@ func TestKeys(t *testing.T) { copy(addr, network.RandomAddr().Over()) outkey := network.RandomAddr().Over() topicobj := BytesToTopic([]byte("foo:42")) - ps.SetPeerPublicKey(&theirprivkey.PublicKey, topicobj, &addr) - outkeyid, err := ps.SetSymmetricKey(outkey, topicobj, &addr, false) + ps.SetPeerPublicKey(&theirprivkey.PublicKey, topicobj, addr) + outkeyid, err := ps.SetSymmetricKey(outkey, topicobj, addr, false) if err != nil { t.Fatalf("failed to set 'our' outgoing symmetric key") } // make a symmetric key that we will send to peer for encrypting messages to us - inkeyid, err := ps.GenerateSymmetricKey(topicobj, &addr, true) + inkeyid, err := ps.GenerateSymmetricKey(topicobj, addr, true) if err != nil { t.Fatalf("failed to set 'our' incoming symmetric key") } @@ -816,8 +816,8 @@ func TestKeys(t *testing.T) { // check that the key is stored in the peerpool psp := ps.symKeyPool[inkeyid][topicobj] - if psp.address != &addr { - t.Fatalf("inkey address does not match; %p != %p", psp.address, &addr) + if !bytes.Equal(psp.address, addr) { + t.Fatalf("inkey address does not match; %p != %p", psp.address, addr) } } @@ -1008,6 +1008,34 @@ func TestRawAllow(t *testing.T) { } } +// BELOW HERE ARE TESTS USING THE SIMULATION FRAMEWORK + +// tests that the API layer can handle edge case values +func TestApi(t *testing.T) { + clients, err := setupNetwork(2, true) + if err != nil { + t.Fatal(err) + } + + topic := "0xdeadbeef" + + err = clients[0].Call(nil, "pss_sendRaw", "0x", topic, "0x666f6f") + if err != nil { + t.Fatal(err) + } + + err = clients[0].Call(nil, "pss_sendRaw", "0xabcdef", topic, "0x") + if err == nil { + t.Fatal("expected error on empty msg") + } + + overflowAddr := [33]byte{} + err = clients[0].Call(nil, "pss_sendRaw", hexutil.Encode(overflowAddr[:]), topic, "0x666f6f") + if err == nil { + t.Fatal("expected error on send too big address") + } +} + // verifies that nodes can send and receive raw (verbatim) messages func TestSendRaw(t *testing.T) { t.Run("32", testSendRaw) @@ -1668,7 +1696,7 @@ func benchmarkSymKeySend(b *testing.B) { topic := BytesToTopic([]byte("foo")) to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - symkeyid, err := ps.GenerateSymmetricKey(topic, &to, true) + symkeyid, err := ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("could not generate symkey: %v", err) } @@ -1676,7 +1704,7 @@ func benchmarkSymKeySend(b *testing.B) { if err != nil { b.Fatalf("could not retrieve symkey: %v", err) } - ps.SetSymmetricKey(symkey, topic, &to, false) + ps.SetSymmetricKey(symkey, topic, to, false) b.ResetTimer() for i := 0; i < b.N; i++ { @@ -1712,7 +1740,7 @@ func benchmarkAsymKeySend(b *testing.B) { topic := BytesToTopic([]byte("foo")) to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - ps.SetPeerPublicKey(&privkey.PublicKey, topic, &to) + ps.SetPeerPublicKey(&privkey.PublicKey, topic, to) b.ResetTimer() for i := 0; i < b.N; i++ { ps.SendAsym(common.ToHex(crypto.FromECDSAPub(&privkey.PublicKey)), topic, msg) @@ -1761,7 +1789,7 @@ func benchmarkSymkeyBruteforceChangeaddr(b *testing.B) { for i := 0; i < int(keycount); i++ { to := make(PssAddress, 32) copy(to[:], network.RandomAddr().Over()) - keyid, err = ps.GenerateSymmetricKey(topic, &to, true) + keyid, err = ps.GenerateSymmetricKey(topic, to, true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) } @@ -1843,7 +1871,7 @@ func benchmarkSymkeyBruteforceSameaddr(b *testing.B) { topic := BytesToTopic([]byte("foo")) for i := 0; i < int(keycount); i++ { copy(addr[i], network.RandomAddr().Over()) - keyid, err = ps.GenerateSymmetricKey(topic, &addr[i], true) + keyid, err = ps.GenerateSymmetricKey(topic, addr[i], true) if err != nil { b.Fatalf("cant generate symkey #%d: %v", i, err) } @@ -2044,12 +2072,13 @@ func NewAPITest(ps *Pss) *APITest { return &APITest{Pss: ps} } -func (apitest *APITest) SetSymKeys(pubkeyid string, recvsymkey []byte, sendsymkey []byte, limit uint16, topic Topic, to PssAddress) ([2]string, error) { - recvsymkeyid, err := apitest.SetSymmetricKey(recvsymkey, topic, &to, true) +func (apitest *APITest) SetSymKeys(pubkeyid string, recvsymkey []byte, sendsymkey []byte, limit uint16, topic Topic, to hexutil.Bytes) ([2]string, error) { + + recvsymkeyid, err := apitest.SetSymmetricKey(recvsymkey, topic, PssAddress(to), true) if err != nil { return [2]string{}, err } - sendsymkeyid, err := apitest.SetSymmetricKey(sendsymkey, topic, &to, false) + sendsymkeyid, err := apitest.SetSymmetricKey(sendsymkey, topic, PssAddress(to), false) if err != nil { return [2]string{}, err } From fe86a707d8270108e0fb77359e702689664dcb4b Mon Sep 17 00:00:00 2001 From: Javier Peletier Date: Tue, 18 Dec 2018 15:25:02 +0100 Subject: [PATCH 68/68] swarm/storage: remove unused methods from Chunk interface (#18283) --- swarm/storage/common_test.go | 5 +++-- swarm/storage/memstore.go | 2 +- swarm/storage/types.go | 18 ------------------ 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/swarm/storage/common_test.go b/swarm/storage/common_test.go index af104a5aeb..bcc29d8cc7 100644 --- a/swarm/storage/common_test.go +++ b/swarm/storage/common_test.go @@ -179,8 +179,9 @@ func testStoreCorrect(m ChunkStore, n int, chunksize int64, t *testing.T) { return fmt.Errorf("key does not match retrieved chunk Address") } hasher := MakeHashFunc(DefaultHash)() - hasher.ResetWithLength(chunk.SpanBytes()) - hasher.Write(chunk.Payload()) + data := chunk.Data() + hasher.ResetWithLength(data[:8]) + hasher.Write(data[8:]) exp := hasher.Sum(nil) if !bytes.Equal(h, exp) { return fmt.Errorf("key is not hash of chunk data") diff --git a/swarm/storage/memstore.go b/swarm/storage/memstore.go index 36b1e00d9b..86e5813d1b 100644 --- a/swarm/storage/memstore.go +++ b/swarm/storage/memstore.go @@ -57,7 +57,7 @@ func (m *MemStore) Get(_ context.Context, addr Address) (Chunk, error) { if !ok { return nil, ErrChunkNotFound } - return c.(*chunk), nil + return c.(Chunk), nil } func (m *MemStore) Put(_ context.Context, c Chunk) error { diff --git a/swarm/storage/types.go b/swarm/storage/types.go index e5c5f89a09..322d95c47a 100644 --- a/swarm/storage/types.go +++ b/swarm/storage/types.go @@ -184,9 +184,6 @@ func (c AddressCollection) Swap(i, j int) { // Chunk interface implemented by context.Contexts and data chunks type Chunk interface { Address() Address - Payload() []byte - SpanBytes() []byte - Span() int64 Data() []byte } @@ -208,25 +205,10 @@ func (c *chunk) Address() Address { return c.addr } -func (c *chunk) SpanBytes() []byte { - return c.sdata[:8] -} - -func (c *chunk) Span() int64 { - if c.span == -1 { - c.span = int64(binary.LittleEndian.Uint64(c.sdata[:8])) - } - return c.span -} - func (c *chunk) Data() []byte { return c.sdata } -func (c *chunk) Payload() []byte { - return c.sdata[8:] -} - // String() for pretty printing func (self *chunk) String() string { return fmt.Sprintf("Address: %v TreeSize: %v Chunksize: %v", self.addr.Log(), self.span, len(self.sdata))