diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c5ae550e34..219996d466 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -19,6 +19,7 @@ package main import ( "encoding/json" "fmt" + "github.com/ethereum/go-ethereum/core/state/snapshot" "os" "path/filepath" "runtime" @@ -201,6 +202,22 @@ Use "ethereum dump 0" to dump the genesis block.`, }, Category: "BLOCKCHAIN COMMANDS", } + generateTrieCommand = cli.Command{ + Action: utils.MigrateFlags(snapToHash), + Name: "snaphash", + Usage: "Calculate the trie root hash from the snapshot db", + ArgsUsage: " ", + Flags: []cli.Flag{ + utils.DataDirFlag, + utils.AncientFlag, + utils.CacheFlag, + utils.TestnetFlag, + utils.RinkebyFlag, + utils.GoerliFlag, + utils.SyncModeFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + } ) // initGenesis will initialise the given JSON format genesis file and writes it as @@ -576,6 +593,40 @@ func inspect(ctx *cli.Context) error { return rawdb.InspectDatabase(chainDb) } +func snapToHash(ctx *cli.Context) error { + node, _ := makeConfigNode(ctx) + chain, chainDb := utils.MakeChain(ctx, node) + + defer func() { + node.Close() + chain.Stop() + chainDb.Close() + }() + + snapTree := chain.Snapshot() + if snapTree == nil { + return fmt.Errorf("No snapshot tree available") + } + block := chain.CurrentBlock() + if block == nil { + return fmt.Errorf("no blocks present") + } + root := block.Root() + it, err := snapTree.AccountIterator(root, common.Hash{}) + if err != nil { + return fmt.Errorf("Could not create iterator for root %x: %v", root, err) + } + generatedRoot := snapshot.GenerateTrieRoot(it) + if err := it.Error(); err != nil { + fmt.Printf("Iterator error: %v\n", it.Error()) + } + if root != generatedRoot { + return fmt.Errorf("Wrong hash generated, expected %x, got %x", root, generatedRoot[:]) + } + log.Info("Generation done", "root", generatedRoot) + return nil +} + // hashish returns true for strings that look like hashes. func hashish(x string) bool { _, err := strconv.Atoi(x) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 3615a91660..5056c514c4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -210,6 +210,7 @@ func init() { dumpCommand, dumpGenesisCommand, inspectCommand, + generateTrieCommand, // See accountcmd.go: accountCommand, walletCommand, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index cbbe530701..7cadf9c2d7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1740,10 +1740,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai TrieDirtyDisabled: ctx.GlobalString(GCModeFlag.Name) == "archive", TrieTimeLimit: eth.DefaultConfig.TrieTimeout, SnapshotLimit: eth.DefaultConfig.SnapshotCache, + SnapshotWait: false, } if !ctx.GlobalIsSet(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled } + // Cover your eyes, this is a hack + if ctx.Command.Name == "snaphash" { + cache.SnapshotLimit = eth.DefaultConfig.SnapshotCache + cache.SnapshotWait = true + } if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) { cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100 } diff --git a/core/blockchain.go b/core/blockchain.go index de0d4f3993..2f082789fb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -313,6 +313,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par return bc, nil } +func (bc *BlockChain) Snapshot() *snapshot.Tree { + return bc.snaps +} + func (bc *BlockChain) getProcInterrupt() bool { return atomic.LoadInt32(&bc.procInterrupt) == 1 } diff --git a/core/state/snapshot/account.go b/core/state/snapshot/account.go index 1068dc2a01..496e1a39df 100644 --- a/core/state/snapshot/account.go +++ b/core/state/snapshot/account.go @@ -52,3 +52,19 @@ func AccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byt } return data } + +func SlimToFull(data []byte) []byte { + acc := &Account{} + rlp.DecodeBytes(data, acc) + if len(acc.Root) == 0 { + acc.Root = emptyRoot[:] + } + if len(acc.CodeHash) == 0 { + acc.CodeHash = emptyCode[:] + } + fullData, err := rlp.EncodeToBytes(acc) + if err != nil { + panic(err) + } + return fullData +} diff --git a/core/state/snapshot/hextrie_generator.go b/core/state/snapshot/hextrie_generator.go index a8d1b67004..d882bd6cb2 100644 --- a/core/state/snapshot/hextrie_generator.go +++ b/core/state/snapshot/hextrie_generator.go @@ -17,10 +17,13 @@ package snapshot import ( + "sync" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" - "sync" ) type leaf struct { @@ -32,7 +35,8 @@ type trieGeneratorFn func(in chan (leaf), out chan (common.Hash)) // GenerateTrieRoot takes an account iterator and reproduces the root hash. func GenerateTrieRoot(it AccountIterator) common.Hash { - return generateTrieRoot(it, StackGenerate) + //return generateTrieRoot(it, StackGenerate) + return generateTrieRoot(it, StdGenerate) } func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash { @@ -47,11 +51,24 @@ func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Ha wg.Done() }() // Feed leaves + start := time.Now() + logged := time.Now() + accounts := 0 for it.Next() { - in <- leaf{it.Hash(), it.Account()} + slimData := it.Account() + fullData := SlimToFull(slimData) + l := leaf{it.Hash(), fullData} + in <- l + if time.Since(logged) > 8*time.Second { + log.Info("Generating trie hash from snapshot", + "at", l.key, "accounts", accounts, "elapsed", time.Since(start)) + logged = time.Now() + } + accounts++ } close(in) result := <-out + log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start)) wg.Wait() return result } diff --git a/core/state/snapshot/trie_generator_test.go b/core/state/snapshot/trie_generator_test.go index 412805685b..4ddb0c81b3 100644 --- a/core/state/snapshot/trie_generator_test.go +++ b/core/state/snapshot/trie_generator_test.go @@ -18,6 +18,8 @@ package snapshot import ( "encoding/binary" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/trie" "math/rand" "testing" @@ -229,3 +231,24 @@ func BenchmarkTrieGeneration(b *testing.B) { }) }) } + +func TestStackVsStandard(t *testing.T) { + type kv struct { + key string + value string + } + vals := []kv{ + {key: "04f0860f1d82f4f0e61a03038cb0ffc08d15e22cb3d91d902c8acc32fa709b95", value: "f8440180a08e762c2b29fb1357d0794271a4dbe16167d8b28f1792ad9f78cad08206816127a010b37de11f39e0a372615c70e1d4d7c613937e8f61823d59be9bea62112e175c"}, + {key: "04f0862f9177d381deeed0e6af3b0751f3cce6887746ba13cf41aa1c4dbf6591", value: "f8440180a014baf10561054a68fe522434b4d4c25e1b377e745bf1d676afa71bc891cacf9ba0debc58a981ca4f637e282ab5985d169a0237d03ea9336bc3434d9dce79e62ab3"}, + {key: "04f0a6c0cb97e624bcb799f7d88717fe7fe4894877a8987a27d4792c36a2833e", value: "f8440180a0880595df1b6b3923e8036106cb641aae6b1249faa02d3217da8c556c0fff172ba06569f607421e3779a571977d84910e1177059946e0a064e487b1502e6a282623"}, + } + stackT := trie.NewStackTrie() + stdT, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New())) + for _, kv := range vals { + stackT.TryUpdate(common.FromHex(kv.key), common.FromHex(kv.value)) + stdT.TryUpdate(common.FromHex(kv.key), common.FromHex(kv.value)) + } + if got, exp := stackT.Hash(), stdT.Hash(); got != exp { + t.Errorf("Hash mismatch, got %x, exp %x", got, exp) + } +}