From 2977538ac0c1157cc140c2df9b0c54e00441d935 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 11 Jun 2018 09:38:05 +0200 Subject: [PATCH 1/9] light: new CHTs for mainnet and ropsten (#16926) --- light/postprocess.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/light/postprocess.go b/light/postprocess.go index c06c18027e..121321ae66 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -59,18 +59,18 @@ type trustedCheckpoint struct { var ( mainnetCheckpoint = trustedCheckpoint{ name: "mainnet", - sectionIdx: 170, - sectionHead: common.HexToHash("3bb2c28bcce463d57968f14f56cdb3fbf35349ab7a701f44c1afb57349c9a356"), - chtRoot: common.HexToHash("d92b6d0853455f8439086292338e87f69781921680dd7aa072fb71547b87415e"), - bloomTrieRoot: common.HexToHash("e4e8250a2fefddead7ae42daecd848cbf9b66d748a8270f8bbd4370b764bb9e9"), + sectionIdx: 174, + sectionHead: common.HexToHash("a3ef48cd8f1c3a08419f0237fc7763491fe89497b3144b17adf87c1c43664613"), + chtRoot: common.HexToHash("dcbeed9f4dea1b3cb75601bb27c51b9960c28e5850275402ac49a150a667296e"), + bloomTrieRoot: common.HexToHash("6b7497a4a03e33870a2383cb6f5e70570f12b1bf5699063baf8c71d02ca90b02"), } ropstenCheckpoint = trustedCheckpoint{ name: "ropsten", - sectionIdx: 97, - sectionHead: common.HexToHash("719448c67c01eb5b9f27833a36a4e34612f66801316d7ff37daf9e77fb4cd095"), - chtRoot: common.HexToHash("a7857afc15930ca6e583b6c3d563a025144011655843d52d28e2fdaadd417bea"), - bloomTrieRoot: common.HexToHash("9c71d4b50cbec86dfeaa8e08992de8a4667b81d13c54d6522b17ce2fc5d36416"), + sectionIdx: 102, + sectionHead: common.HexToHash("9017ab08465cb2b2dee035ee5b817bbd7fa28e2c8d2cd903e0aed1cccb249e89"), + chtRoot: common.HexToHash("f61c10a7a787a5ef15f0ae1ae6c13c64331e57e79d0466d2bd9b0c06833fe956"), + bloomTrieRoot: common.HexToHash("69f2ad19aa46d5213a90137b3d2c9bff8a7c9483f7170f0125096ff450c9a873"), } ) From 69c52bde3f5e48a3b74264bf4854e9768ede75b2 Mon Sep 17 00:00:00 2001 From: Steven Roose Date: Mon, 11 Jun 2018 09:41:09 +0200 Subject: [PATCH 2/9] ethclient: fix RPC parse error of Parity response (#16924) The error produced when using a Parity RPC was the following: ERROR: transaction did not get mined: failed to get tx for txid 0xbdeb094b3278019383c8da148ff1cb5b5dbd61bf8731bc2310ac1b8ed0235226: json: cannot unmarshal non-string into Go struct field txExtraInfo.blockHash of type common.Hash --- ethclient/ethclient.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index b482245878..b40837c8cc 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -141,7 +141,9 @@ func (ec *Client) getBlock(ctx context.Context, method string, args ...interface // Fill the sender cache of transactions in the block. txs := make([]*types.Transaction, len(body.Transactions)) for i, tx := range body.Transactions { - setSenderFromServer(tx.tx, tx.From, body.Hash) + if tx.From != nil { + setSenderFromServer(tx.tx, *tx.From, body.Hash) + } txs[i] = tx.tx } return types.NewBlockWithHeader(head).WithBody(txs, uncles), nil @@ -174,9 +176,9 @@ type rpcTransaction struct { } type txExtraInfo struct { - BlockNumber *string - BlockHash common.Hash - From common.Address + BlockNumber *string `json:"blockNumber,omitempty"` + BlockHash *common.Hash `json:"blockHash,omitempty"` + From *common.Address `json:"from,omitempty"` } func (tx *rpcTransaction) UnmarshalJSON(msg []byte) error { @@ -197,7 +199,9 @@ func (ec *Client) TransactionByHash(ctx context.Context, hash common.Hash) (tx * } else if _, r, _ := json.tx.RawSignatureValues(); r == nil { return nil, false, fmt.Errorf("server returned transaction without signature") } - setSenderFromServer(json.tx, json.From, json.BlockHash) + if json.From != nil && json.BlockHash != nil { + setSenderFromServer(json.tx, *json.From, *json.BlockHash) + } return json.tx, json.BlockNumber == nil, nil } @@ -244,7 +248,9 @@ func (ec *Client) TransactionInBlock(ctx context.Context, blockHash common.Hash, return nil, fmt.Errorf("server returned transaction without signature") } } - setSenderFromServer(json.tx, json.From, json.BlockHash) + if json.From != nil && json.BlockHash != nil { + setSenderFromServer(json.tx, *json.From, *json.BlockHash) + } return json.tx, err } From eac16f98243206bee99c3daaa8aef08695f5b69e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 11 Jun 2018 10:03:40 +0200 Subject: [PATCH 3/9] core: improve getBadBlocks to return full block rlp (#16902) * core: improve getBadBlocks to return full block rlp * core, eth, ethapi: changes to getBadBlocks formatting * ethapi: address review concerns --- core/blockchain.go | 20 +++++++------------- eth/api.go | 29 +++++++++++++++++++++++++++-- internal/ethapi/api.go | 20 ++++++++++++++------ 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index bf1bbe6cb7..ea26fa0345 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1392,27 +1392,21 @@ func (bc *BlockChain) update() { } } -// BadBlockArgs represents the entries in the list returned when bad blocks are queried. -type BadBlockArgs struct { - Hash common.Hash `json:"hash"` - Header *types.Header `json:"header"` -} - // BadBlocks returns a list of the last 'bad blocks' that the client has seen on the network -func (bc *BlockChain) BadBlocks() ([]BadBlockArgs, error) { - headers := make([]BadBlockArgs, 0, bc.badBlocks.Len()) +func (bc *BlockChain) BadBlocks() []*types.Block { + blocks := make([]*types.Block, 0, bc.badBlocks.Len()) for _, hash := range bc.badBlocks.Keys() { - if hdr, exist := bc.badBlocks.Peek(hash); exist { - header := hdr.(*types.Header) - headers = append(headers, BadBlockArgs{header.Hash(), header}) + if blk, exist := bc.badBlocks.Peek(hash); exist { + block := blk.(*types.Block) + blocks = append(blocks, block) } } - return headers, nil + return blocks } // addBadBlock adds a bad block to the bad-block LRU cache func (bc *BlockChain) addBadBlock(block *types.Block) { - bc.badBlocks.Add(block.Header().Hash(), block.Header()) + bc.badBlocks.Add(block.Hash(), block) } // reportBlock logs a bad block error. diff --git a/eth/api.go b/eth/api.go index 247ca7485c..446161cc4a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/params" @@ -351,10 +352,34 @@ func (api *PrivateDebugAPI) Preimage(ctx context.Context, hash common.Hash) (hex return nil, errors.New("unknown preimage") } +// BadBlockArgs represents the entries in the list returned when bad blocks are queried. +type BadBlockArgs struct { + Hash common.Hash `json:"hash"` + Block map[string]interface{} `json:"block"` + RLP string `json:"rlp"` +} + // GetBadBLocks returns a list of the last 'bad blocks' that the client has seen on the network // and returns them as a JSON list of block-hashes -func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]core.BadBlockArgs, error) { - return api.eth.BlockChain().BadBlocks() +func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) { + blocks := api.eth.BlockChain().BadBlocks() + results := make([]*BadBlockArgs, len(blocks)) + + var err error + for i, block := range blocks { + results[i] = &BadBlockArgs{ + Hash: block.Hash(), + } + if rlpBytes, err := rlp.EncodeToBytes(block); err != nil { + results[i].RLP = err.Error() // Hacky, but hey, it works + } else { + results[i].RLP = fmt.Sprintf("0x%x", rlpBytes) + } + if results[i].Block, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { + results[i].Block = map[string]interface{}{"error": err.Error()} + } + } + return results, nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 87eba7e1a3..7a736bb76d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -792,10 +792,10 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { return formatted } -// rpcOutputBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are +// RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { +func RPCMarshalBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { head := b.Header() // copies the header once fields := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), @@ -808,7 +808,6 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx "stateRoot": head.Root, "miner": head.Coinbase, "difficulty": (*hexutil.Big)(head.Difficulty), - "totalDifficulty": (*hexutil.Big)(s.b.GetTd(b.Hash())), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(b.Size()), "gasLimit": hexutil.Uint64(head.GasLimit), @@ -822,17 +821,15 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx formatTx := func(tx *types.Transaction) (interface{}, error) { return tx.Hash(), nil } - if fullTx { formatTx = func(tx *types.Transaction) (interface{}, error) { return newRPCTransactionFromBlockHash(b, tx.Hash()), nil } } - txs := b.Transactions() transactions := make([]interface{}, len(txs)) var err error - for i, tx := range b.Transactions() { + for i, tx := range txs { if transactions[i], err = formatTx(tx); err != nil { return nil, err } @@ -850,6 +847,17 @@ func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx return fields, nil } +// rpcOutputBlock uses the generalized output filler, then adds the total difficulty field, which requires +// a `PublicBlockchainAPI`. +func (s *PublicBlockChainAPI) rpcOutputBlock(b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields, err := RPCMarshalBlock(b, inclTx, fullTx) + if err != nil { + return nil, err + } + fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(b.Hash())) + return fields, err +} + // RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction type RPCTransaction struct { BlockHash common.Hash `json:"blockHash"` From 1d666cf27ec366a967d9afa0e8a370cb4cf33481 Mon Sep 17 00:00:00 2001 From: xincaosu Date: Mon, 11 Jun 2018 17:01:13 +0800 Subject: [PATCH 4/9] rpc: fix a comment typo (#16929) --- rpc/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rpc/server.go b/rpc/server.go index 7f304d8d93..2b7393dd20 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -145,7 +145,7 @@ func (s *Server) serveRequest(ctx context.Context, codec ServerCodec, singleShot defer cancel() // if the codec supports notification include a notifier that callbacks can use - // to send notification to clients. It is thight to the codec/connection. If the + // to send notification to clients. It is tied to the codec/connection. If the // connection is closed the notifier will stop and cancels all active subscriptions. if options&OptionSubscriptions == OptionSubscriptions { ctx = context.WithValue(ctx, notifierKey{}, newNotifier(codec)) From 99483e85b92e07078291442bf1cd4c0db22a262d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Jun 2018 13:15:59 +0300 Subject: [PATCH 5/9] rpc: support returning nil pointer big.Ints (null) --- internal/ethapi/api.go | 14 +++++++------- rpc/json.go | 8 -------- rpc/server.go | 1 - rpc/utils.go | 15 --------------- 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 7a736bb76d..f95388d658 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -62,8 +62,9 @@ func NewPublicEthereumAPI(b Backend) *PublicEthereumAPI { } // GasPrice returns a suggestion for a gas price. -func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*big.Int, error) { - return s.b.SuggestPrice(ctx) +func (s *PublicEthereumAPI) GasPrice(ctx context.Context) (*hexutil.Big, error) { + price, err := s.b.SuggestPrice(ctx) + return (*hexutil.Big)(price), err } // ProtocolVersion returns the current Ethereum protocol version this node supports @@ -487,21 +488,20 @@ func NewPublicBlockChainAPI(b Backend) *PublicBlockChainAPI { } // BlockNumber returns the block number of the chain head. -func (s *PublicBlockChainAPI) BlockNumber() *big.Int { +func (s *PublicBlockChainAPI) BlockNumber() hexutil.Uint64 { header, _ := s.b.HeaderByNumber(context.Background(), rpc.LatestBlockNumber) // latest header should always be available - return header.Number + return hexutil.Uint64(header.Number.Uint64()) } // GetBalance returns the amount of wei for the given address in the state of the // given block number. The rpc.LatestBlockNumber and rpc.PendingBlockNumber meta // block numbers are also allowed. -func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*big.Int, error) { +func (s *PublicBlockChainAPI) GetBalance(ctx context.Context, address common.Address, blockNr rpc.BlockNumber) (*hexutil.Big, error) { state, _, err := s.b.StateAndHeaderByNumber(ctx, blockNr) if state == nil || err != nil { return nil, err } - b := state.GetBalance(address) - return b, state.Error() + return (*hexutil.Big)(state.GetBalance(address)), state.Error() } // GetBlockByNumber returns the requested block. When blockNr is -1 the chain head is returned. When fullTx is true all diff --git a/rpc/json.go b/rpc/json.go index 837011f51b..a523eeb8ef 100644 --- a/rpc/json.go +++ b/rpc/json.go @@ -320,9 +320,6 @@ func parsePositionalArguments(rawArgs json.RawMessage, types []reflect.Type) ([] // CreateResponse will create a JSON-RPC success response with the given id and reply as result. func (c *jsonCodec) CreateResponse(id interface{}, reply interface{}) interface{} { - if isHexNum(reflect.TypeOf(reply)) { - return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: fmt.Sprintf(`%#x`, reply)} - } return &jsonSuccessResponse{Version: jsonrpcVersion, Id: id, Result: reply} } @@ -340,11 +337,6 @@ func (c *jsonCodec) CreateErrorResponseWithInfo(id interface{}, err Error, info // CreateNotification will create a JSON-RPC notification with the given subscription id and event as params. func (c *jsonCodec) CreateNotification(subid, namespace string, event interface{}) interface{} { - if isHexNum(reflect.TypeOf(event)) { - return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, - Params: jsonSubscription{Subscription: subid, Result: fmt.Sprintf(`%#x`, event)}} - } - return &jsonNotification{Version: jsonrpcVersion, Method: namespace + notificationMethodSuffix, Params: jsonSubscription{Subscription: subid, Result: event}} } diff --git a/rpc/server.go b/rpc/server.go index 2b7393dd20..8925419fe0 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -313,7 +313,6 @@ func (s *Server) handle(ctx context.Context, codec ServerCodec, req *serverReque if len(reply) == 0 { return codec.CreateResponse(req.id, nil), nil } - if req.callb.errPos >= 0 { // test if method returned an error if !reply[req.callb.errPos].IsNil() { e := reply[req.callb.errPos].Interface().(error) diff --git a/rpc/utils.go b/rpc/utils.go index 9315cab591..7f7ac4520b 100644 --- a/rpc/utils.go +++ b/rpc/utils.go @@ -22,7 +22,6 @@ import ( crand "crypto/rand" "encoding/binary" "encoding/hex" - "math/big" "math/rand" "reflect" "strings" @@ -105,20 +104,6 @@ func formatName(name string) string { return string(ret) } -var bigIntType = reflect.TypeOf((*big.Int)(nil)).Elem() - -// Indication if this type should be serialized in hex -func isHexNum(t reflect.Type) bool { - if t == nil { - return false - } - for t.Kind() == reflect.Ptr { - t = t.Elem() - } - - return t == bigIntType -} - // suitableCallbacks iterates over the methods of the given type. It will determine if a method satisfies the criteria // for a RPC callback or a subscription callback and adds it to the collection of callbacks or subscriptions. See server // documentation for a summary of these criteria. From a3267ed9296b08ec170b239e519cb5aff6ee25f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 11 Jun 2018 14:32:13 +0300 Subject: [PATCH 6/9] trie: don't report the root flushlist as an alloc --- trie/database.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trie/database.go b/trie/database.go index 76a6cf79db..468c139df4 100644 --- a/trie/database.go +++ b/trie/database.go @@ -297,7 +297,7 @@ func (db *Database) Cap(limit common.StorageSize) error { // db.nodesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - size := db.nodesSize + common.StorageSize(len(db.nodes)*2*common.HashLength) + size := db.nodesSize + common.StorageSize((len(db.nodes)-1)*2*common.HashLength) // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. @@ -512,6 +512,6 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // db.nodesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be // counted. For every useful node, we track 2 extra hashes as the flushlist. - var flushlistSize = common.StorageSize(len(db.nodes) * 2 * common.HashLength) + var flushlistSize = common.StorageSize((len(db.nodes) - 1) * 2 * common.HashLength) return db.nodesSize + flushlistSize, db.preimagesSize } From b487bdf0ba21f1877501a9bdd97f84701273e022 Mon Sep 17 00:00:00 2001 From: Clayton Jacobs Date: Mon, 11 Jun 2018 19:45:25 +0800 Subject: [PATCH 7/9] metrics: removed repetitive calculations (#16944) --- metrics/metrics.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/metrics/metrics.go b/metrics/metrics.go index e24324814c..11203ccdc8 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -68,17 +68,20 @@ func CollectProcessMetrics(refresh time.Duration) { } // Iterate loading the different stats and updating the meters for i := 1; ; i++ { - runtime.ReadMemStats(memstats[i%2]) - memAllocs.Mark(int64(memstats[i%2].Mallocs - memstats[(i-1)%2].Mallocs)) - memFrees.Mark(int64(memstats[i%2].Frees - memstats[(i-1)%2].Frees)) - memInuse.Mark(int64(memstats[i%2].Alloc - memstats[(i-1)%2].Alloc)) - memPauses.Mark(int64(memstats[i%2].PauseTotalNs - memstats[(i-1)%2].PauseTotalNs)) + location1 := i%2 + location2 := (i-1)%2 - if ReadDiskStats(diskstats[i%2]) == nil { - diskReads.Mark(diskstats[i%2].ReadCount - diskstats[(i-1)%2].ReadCount) - diskReadBytes.Mark(diskstats[i%2].ReadBytes - diskstats[(i-1)%2].ReadBytes) - diskWrites.Mark(diskstats[i%2].WriteCount - diskstats[(i-1)%2].WriteCount) - diskWriteBytes.Mark(diskstats[i%2].WriteBytes - diskstats[(i-1)%2].WriteBytes) + runtime.ReadMemStats(memstats[location1]) + memAllocs.Mark(int64(memstats[location1].Mallocs - memstats[location2].Mallocs)) + memFrees.Mark(int64(memstats[location1].Frees - memstats[location2].Frees)) + memInuse.Mark(int64(memstats[location1].Alloc - memstats[location2].Alloc)) + memPauses.Mark(int64(memstats[location1].PauseTotalNs - memstats[location2].PauseTotalNs)) + + if ReadDiskStats(diskstats[location1]) == nil { + diskReads.Mark(diskstats[location1].ReadCount - diskstats[location2].ReadCount) + diskReadBytes.Mark(diskstats[location1].ReadBytes - diskstats[location2].ReadBytes) + diskWrites.Mark(diskstats[location1].WriteCount - diskstats[location2].WriteCount) + diskWriteBytes.Mark(diskstats[location1].WriteBytes - diskstats[location2].WriteBytes) } time.Sleep(refresh) } From aab7ab04b01acb0327786911e7434090c9afb722 Mon Sep 17 00:00:00 2001 From: Wenbiao Zheng Date: Mon, 11 Jun 2018 21:06:26 +0800 Subject: [PATCH 8/9] core/rawdb: wrap db key creations (#16914) * core/rawdb: use wrappered helper to assemble key * core/rawdb: wrappered helper to assemble key * core/rawdb: rewrite the wrapper, pass common.Hash --- core/rawdb/accessors_chain.go | 48 ++++++++++++---------------- core/rawdb/accessors_indexes.go | 22 +++---------- core/rawdb/accessors_metadata.go | 8 ++--- core/rawdb/schema.go | 55 ++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 48 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index a26a42ba7c..da54328326 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -29,7 +29,7 @@ import ( // ReadCanonicalHash retrieves the hash assigned to a canonical block number. func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)) + data, _ := db.Get(headerHashKey(number)) if len(data) == 0 { return common.Hash{} } @@ -38,22 +38,21 @@ func ReadCanonicalHash(db DatabaseReader, number uint64) common.Hash { // WriteCanonicalHash stores the hash assigned to a canonical block number. func WriteCanonicalHash(db DatabaseWriter, hash common.Hash, number uint64) { - key := append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) - if err := db.Put(key, hash.Bytes()); err != nil { + if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) } } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db DatabaseDeleter, number uint64) { - if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)); err != nil { + if err := db.Delete(headerHashKey(number)); err != nil { log.Crit("Failed to delete number to hash mapping", "err", err) } } // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db DatabaseReader, hash common.Hash) *uint64 { - data, _ := db.Get(append(headerNumberPrefix, hash.Bytes()...)) + data, _ := db.Get(headerNumberKey(hash)) if len(data) != 8 { return nil } @@ -129,14 +128,13 @@ func WriteFastTrieProgress(db DatabaseWriter, count uint64) { // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(headerKey(number, hash)) return data } // HasHeader verifies the existence of a block header corresponding to the hash. func HasHeader(db DatabaseReader, hash common.Hash, number uint64) bool { - key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) - if has, err := db.Has(key); !has || err != nil { + if has, err := db.Has(headerKey(number, hash)); !has || err != nil { return false } return true @@ -161,11 +159,11 @@ func ReadHeader(db DatabaseReader, hash common.Hash, number uint64) *types.Heade func WriteHeader(db DatabaseWriter, header *types.Header) { // Write the hash -> number mapping var ( - hash = header.Hash().Bytes() + hash = header.Hash() number = header.Number.Uint64() encoded = encodeBlockNumber(number) ) - key := append(headerNumberPrefix, hash...) + key := headerNumberKey(hash) if err := db.Put(key, encoded); err != nil { log.Crit("Failed to store hash to number mapping", "err", err) } @@ -174,7 +172,7 @@ func WriteHeader(db DatabaseWriter, header *types.Header) { if err != nil { log.Crit("Failed to RLP encode header", "err", err) } - key = append(append(headerPrefix, encoded...), hash...) + key = headerKey(number, hash) if err := db.Put(key, data); err != nil { log.Crit("Failed to store header", "err", err) } @@ -182,32 +180,30 @@ func WriteHeader(db DatabaseWriter, header *types.Header) { // DeleteHeader removes all block header data associated with a hash. func DeleteHeader(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(headerKey(number, hash)); err != nil { log.Crit("Failed to delete header", "err", err) } - if err := db.Delete(append(headerNumberPrefix, hash.Bytes()...)); err != nil { + if err := db.Delete(headerNumberKey(hash)); err != nil { log.Crit("Failed to delete hash to number mapping", "err", err) } } // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db DatabaseReader, hash common.Hash, number uint64) rlp.RawValue { - data, _ := db.Get(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)) + data, _ := db.Get(blockBodyKey(number, hash)) return data } // WriteBodyRLP stores an RLP encoded block body into the database. func WriteBodyRLP(db DatabaseWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { - key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, rlp); err != nil { + if err := db.Put(blockBodyKey(number, hash), rlp); err != nil { log.Crit("Failed to store block body", "err", err) } } // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db DatabaseReader, hash common.Hash, number uint64) bool { - key := append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if has, err := db.Has(key); !has || err != nil { + if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false } return true @@ -238,14 +234,14 @@ func WriteBody(db DatabaseWriter, hash common.Hash, number uint64, body *types.B // DeleteBody removes all block body data associated with a hash. func DeleteBody(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(blockBodyKey(number, hash)); err != nil { log.Crit("Failed to delete block body", "err", err) } } // ReadTd retrieves a block's total difficulty corresponding to the hash. func ReadTd(db DatabaseReader, hash common.Hash, number uint64) *big.Int { - data, _ := db.Get(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash[:]...), headerTDSuffix...)) + data, _ := db.Get(headerTDKey(number, hash)) if len(data) == 0 { return nil } @@ -263,15 +259,14 @@ func WriteTd(db DatabaseWriter, hash common.Hash, number uint64, td *big.Int) { if err != nil { log.Crit("Failed to RLP encode block total difficulty", "err", err) } - key := append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...) - if err := db.Put(key, data); err != nil { + if err := db.Put(headerTDKey(number, hash), data); err != nil { log.Crit("Failed to store block total difficulty", "err", err) } } // DeleteTd removes all block total difficulty data associated with a hash. func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...), headerTDSuffix...)); err != nil { + if err := db.Delete(headerTDKey(number, hash)); err != nil { log.Crit("Failed to delete block total difficulty", "err", err) } } @@ -279,7 +274,7 @@ func DeleteTd(db DatabaseDeleter, hash common.Hash, number uint64) { // ReadReceipts retrieves all the transaction receipts belonging to a block. func ReadReceipts(db DatabaseReader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice - data, _ := db.Get(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash[:]...)) + data, _ := db.Get(blockReceiptsKey(number, hash)) if len(data) == 0 { return nil } @@ -308,15 +303,14 @@ func WriteReceipts(db DatabaseWriter, hash common.Hash, number uint64, receipts log.Crit("Failed to encode block receipts", "err", err) } // Store the flattened receipt slice - key := append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) - if err := db.Put(key, bytes); err != nil { + if err := db.Put(blockReceiptsKey(number, hash), bytes); err != nil { log.Crit("Failed to store block receipts", "err", err) } } // DeleteReceipts removes all receipt data associated with a block hash. func DeleteReceipts(db DatabaseDeleter, hash common.Hash, number uint64) { - if err := db.Delete(append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...)); err != nil { + if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { log.Crit("Failed to delete block receipts", "err", err) } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 9abad14e0c..4ff7e5bd35 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -17,8 +17,6 @@ package rawdb import ( - "encoding/binary" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -28,7 +26,7 @@ import ( // ReadTxLookupEntry retrieves the positional metadata associated with a transaction // hash to allow retrieving the transaction or receipt by hash. func ReadTxLookupEntry(db DatabaseReader, hash common.Hash) (common.Hash, uint64, uint64) { - data, _ := db.Get(append(txLookupPrefix, hash.Bytes()...)) + data, _ := db.Get(txLookupKey(hash)) if len(data) == 0 { return common.Hash{}, 0, 0 } @@ -53,7 +51,7 @@ func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { if err != nil { log.Crit("Failed to encode transaction lookup entry", "err", err) } - if err := db.Put(append(txLookupPrefix, tx.Hash().Bytes()...), data); err != nil { + if err := db.Put(txLookupKey(tx.Hash()), data); err != nil { log.Crit("Failed to store transaction lookup entry", "err", err) } } @@ -61,7 +59,7 @@ func WriteTxLookupEntries(db DatabaseWriter, block *types.Block) { // DeleteTxLookupEntry removes all transaction data associated with a hash. func DeleteTxLookupEntry(db DatabaseDeleter, hash common.Hash) { - db.Delete(append(txLookupPrefix, hash.Bytes()...)) + db.Delete(txLookupKey(hash)) } // ReadTransaction retrieves a specific transaction from the database, along with @@ -97,23 +95,13 @@ func ReadReceipt(db DatabaseReader, hash common.Hash) (*types.Receipt, common.Ha // ReadBloomBits retrieves the compressed bloom bit vector belonging to the given // section and bit index from the. func ReadBloomBits(db DatabaseReader, bit uint, section uint64, head common.Hash) ([]byte, error) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - return db.Get(key) + return db.Get(bloomBitsKey(bit, section, head)) } // WriteBloomBits stores the compressed bloom bits vector belonging to the given // section and bit index. func WriteBloomBits(db DatabaseWriter, bit uint, section uint64, head common.Hash, bits []byte) { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), head.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - if err := db.Put(key, bits); err != nil { + if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil { log.Crit("Failed to store bloom bits", "err", err) } } diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 73ab983f2b..514328e87b 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -45,7 +45,7 @@ func WriteDatabaseVersion(db DatabaseWriter, version int) { // ReadChainConfig retrieves the consensus settings based on the given genesis hash. func ReadChainConfig(db DatabaseReader, hash common.Hash) *params.ChainConfig { - data, _ := db.Get(append(configPrefix, hash[:]...)) + data, _ := db.Get(configKey(hash)) if len(data) == 0 { return nil } @@ -66,14 +66,14 @@ func WriteChainConfig(db DatabaseWriter, hash common.Hash, cfg *params.ChainConf if err != nil { log.Crit("Failed to JSON encode chain config", "err", err) } - if err := db.Put(append(configPrefix, hash[:]...), data); err != nil { + if err := db.Put(configKey(hash), data); err != nil { log.Crit("Failed to store chain config", "err", err) } } // ReadPreimage retrieves a single preimage of the provided hash. func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { - data, _ := db.Get(append(preimagePrefix, hash.Bytes()...)) + data, _ := db.Get(preimageKey(hash)) return data } @@ -81,7 +81,7 @@ func ReadPreimage(db DatabaseReader, hash common.Hash) []byte { // current block number, and is used for debug messages only. func WritePreimages(db DatabaseWriter, number uint64, preimages map[common.Hash][]byte) { for hash, preimage := range preimages { - if err := db.Put(append(preimagePrefix, hash.Bytes()...), preimage); err != nil { + if err := db.Put(preimageKey(hash), preimage); err != nil { log.Crit("Failed to store trie preimage", "err", err) } } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index a4b1596fde..ef597ef309 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -77,3 +77,58 @@ func encodeBlockNumber(number uint64) []byte { binary.BigEndian.PutUint64(enc, number) return enc } + +// headerKey = headerPrefix + num (uint64 big endian) + hash +func headerKey(number uint64, hash common.Hash) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix +func headerTDKey(number uint64, hash common.Hash) []byte { + return append(headerKey(number, hash), headerTDSuffix...) +} + +// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix +func headerHashKey(number uint64) []byte { + return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...) +} + +// headerNumberKey = headerNumberPrefix + hash +func headerNumberKey(hash common.Hash) []byte { + return append(headerNumberPrefix, hash.Bytes()...) +} + +// blockBodyKey = blockBodyPrefix + num (uint64 big endian) + hash +func blockBodyKey(number uint64, hash common.Hash) []byte { + return append(append(blockBodyPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// blockReceiptsKey = blockReceiptsPrefix + num (uint64 big endian) + hash +func blockReceiptsKey(number uint64, hash common.Hash) []byte { + return append(append(blockReceiptsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + +// txLookupKey = txLookupPrefix + hash +func txLookupKey(hash common.Hash) []byte { + return append(txLookupPrefix, hash.Bytes()...) +} + +// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash +func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte { + key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...) + + binary.BigEndian.PutUint16(key[1:], uint16(bit)) + binary.BigEndian.PutUint64(key[3:], section) + + return key +} + +// preimageKey = preimagePrefix + hash +func preimageKey(hash common.Hash) []byte { + return append(preimagePrefix, hash.Bytes()...) +} + +// configKey = configPrefix + hash +func configKey(hash common.Hash) []byte { + return append(configPrefix, hash.Bytes()...) +} From f991995918899c48cc89c4d4ec4329bb4a7ce108 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 11 Jun 2018 21:11:48 +0800 Subject: [PATCH 9/9] ethdb: gracefullly handle quit channel (#16794) * ethdb: gratefullly handle quit channel * ethdb: minor polish --- ethdb/database.go | 74 +++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/ethdb/database.go b/ethdb/database.go index 86dd210762..f4a5ce2c8d 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -141,6 +141,7 @@ func (db *LDBDatabase) Close() { if err := <-errc; err != nil { db.log.Error("Metrics collection failed", "err", err) } + db.quitChan = nil } err := db.db.Close() if err == nil { @@ -189,7 +190,7 @@ func (db *LDBDatabase) Meter(prefix string) { // 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 // // This is how the write delay look like (currently): -// DelayN:5 Delay:406.604657ms +// DelayN:5 Delay:406.604657ms Paused: false // // This is how the iostats look like (currently): // Read(MB):3895.04860 Write(MB):3654.64712 @@ -210,13 +211,19 @@ func (db *LDBDatabase) meter(refresh time.Duration) { lastWritePaused time.Time ) + var ( + errc chan error + merr error + ) + // Iterate ad infinitum and collect the stats - for i := 1; ; i++ { + for i := 1; errc == nil && merr == nil; i++ { // Retrieve the database stats stats, err := db.db.GetProperty("leveldb.stats") if err != nil { db.log.Error("Failed to read database stats", "err", err) - return + merr = err + continue } // Find the compaction table, skip the header lines := strings.Split(stats, "\n") @@ -225,7 +232,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { } if len(lines) <= 3 { db.log.Error("Compaction table not found") - return + merr = errors.New("compaction table not found") + continue } lines = lines[3:] @@ -242,7 +250,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) if err != nil { db.log.Error("Compaction entry parsing failed", "err", err) - return + merr = err + continue } compactions[i%2][idx] += value } @@ -262,7 +271,8 @@ func (db *LDBDatabase) meter(refresh time.Duration) { writedelay, err := db.db.GetProperty("leveldb.writedelay") if err != nil { db.log.Error("Failed to read database write delay statistic", "err", err) - return + merr = err + continue } var ( delayN int64 @@ -272,12 +282,14 @@ func (db *LDBDatabase) meter(refresh time.Duration) { ) if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { db.log.Error("Write delay statistic not found") - return + merr = err + continue } duration, err = time.ParseDuration(delayDuration) if err != nil { db.log.Error("Failed to parse delay duration", "err", err) - return + merr = err + continue } if db.writeDelayNMeter != nil { db.writeDelayNMeter.Mark(delayN - delaystats[0]) @@ -317,53 +329,47 @@ func (db *LDBDatabase) meter(refresh time.Duration) { ioStats, err := db.db.GetProperty("leveldb.iostats") if err != nil { db.log.Error("Failed to read database iostats", "err", err) - return + merr = err + continue } + var nRead, nWrite float64 parts := strings.Split(ioStats, " ") if len(parts) < 2 { db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - return + merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) + continue } - r := strings.Split(parts[0], ":") - if len(r) < 2 { + if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { db.log.Error("Bad syntax of read entry", "entry", parts[0]) - return + merr = err + continue } - read, err := strconv.ParseFloat(r[1], 64) - if err != nil { - db.log.Error("Read entry parsing failed", "err", err) - return - } - w := strings.Split(parts[1], ":") - if len(w) < 2 { + if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { db.log.Error("Bad syntax of write entry", "entry", parts[1]) - return - } - write, err := strconv.ParseFloat(w[1], 64) - if err != nil { - db.log.Error("Write entry parsing failed", "err", err) - return + merr = err + continue } if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((read - iostats[0]) * 1024 * 1024)) + db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) } if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((write - iostats[1]) * 1024 * 1024)) + db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) } - iostats[0] = read - iostats[1] = write + iostats[0], iostats[1] = nRead, nWrite // Sleep a bit, then repeat the stats collection select { - case errc := <-db.quitChan: + case errc = <-db.quitChan: // Quit requesting, stop hammering the database - errc <- nil - return - case <-time.After(refresh): // Timeout, gather a new set of stats } } + + if errc == nil { + errc = <-db.quitChan + } + errc <- merr } func (db *LDBDatabase) NewBatch() Batch {