cmd/geth, core: implement poc trie-root-generator command

cmd/geth/chaincmd.go

@@ -19,6 +19,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -201,6 +202,22 @@ Use "ethereum dump 0" to dump the genesis block.`,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 	}
+	generateTrieCommand = cli.Command{
+		Action:    utils.MigrateFlags(snapToHash),
+		Name:      "snaphash",
+		Usage:     "Calculate the trie root hash from the snapshot db",
+		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.AncientFlag,
+			utils.CacheFlag,
+			utils.TestnetFlag,
+			utils.RinkebyFlag,
+			utils.GoerliFlag,
+			utils.SyncModeFlag,
+		},
+		Category: "BLOCKCHAIN COMMANDS",
+	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -576,6 +593,40 @@ func inspect(ctx *cli.Context) error {
 	return rawdb.InspectDatabase(chainDb)
 }
 
+func snapToHash(ctx *cli.Context) error {
+	node, _ := makeConfigNode(ctx)
+	chain, chainDb := utils.MakeChain(ctx, node)
+
+	defer func() {
+		node.Close()
+		chain.Stop()
+		chainDb.Close()
+	}()
+
+	snapTree := chain.Snapshot()
+	if snapTree == nil {
+		return fmt.Errorf("No snapshot tree available")
+	}
+	block := chain.CurrentBlock()
+	if block == nil {
+		return fmt.Errorf("no blocks present")
+	}
+	root := block.Root()
+	it, err := snapTree.AccountIterator(root, common.Hash{})
+	if err != nil {
+		return fmt.Errorf("Could not create iterator for root %x: %v", root, err)
+	}
+	generatedRoot := snapshot.GenerateTrieRoot(it)
+	if err := it.Error(); err != nil {
+		fmt.Printf("Iterator error: %v\n", it.Error())
+	}
+	if root != generatedRoot {
+		return fmt.Errorf("Wrong hash generated, expected %x, got %x", root, generatedRoot[:])
+	}
+	log.Info("Generation done", "root", generatedRoot)
+	return nil
+}
+
 // hashish returns true for strings that look like hashes.
 func hashish(x string) bool {
 	_, err := strconv.Atoi(x)

cmd/geth/main.go

@@ -210,6 +210,7 @@ func init() {
 		dumpCommand,
 		dumpGenesisCommand,
 		inspectCommand,
+		generateTrieCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

cmd/utils/flags.go

@@ -1740,10 +1740,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
 		TrieDirtyDisabled:   ctx.GlobalString(GCModeFlag.Name) == "archive",
 		TrieTimeLimit:       eth.DefaultConfig.TrieTimeout,
 		SnapshotLimit:       eth.DefaultConfig.SnapshotCache,
+		SnapshotWait:        false,
 	}
 	if !ctx.GlobalIsSet(SnapshotFlag.Name) {
 		cache.SnapshotLimit = 0 // Disabled
 	}
+	// Cover your eyes, this is a hack
+	if ctx.Command.Name == "snaphash" {
+		cache.SnapshotLimit = eth.DefaultConfig.SnapshotCache
+		cache.SnapshotWait = true
+	}
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
 		cache.TrieCleanLimit = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
 	}

core/blockchain.go

@@ -313,6 +313,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
 	return bc, nil
 }
 
+func (bc *BlockChain) Snapshot() *snapshot.Tree {
+	return bc.snaps
+}
+
 func (bc *BlockChain) getProcInterrupt() bool {
 	return atomic.LoadInt32(&bc.procInterrupt) == 1
 }

core/state/snapshot/account.go

@@ -52,3 +52,19 @@ func AccountRLP(nonce uint64, balance *big.Int, root common.Hash, codehash []byt
 	}
 	return data
 }
+
+func SlimToFull(data []byte) []byte {
+	acc := &Account{}
+	rlp.DecodeBytes(data, acc)
+	if len(acc.Root) == 0 {
+		acc.Root = emptyRoot[:]
+	}
+	if len(acc.CodeHash) == 0 {
+		acc.CodeHash = emptyCode[:]
+	}
+	fullData, err := rlp.EncodeToBytes(acc)
+	if err != nil {
+		panic(err)
+	}
+	return fullData
+}

core/state/snapshot/hextrie_generator.go

@@ -17,10 +17,13 @@
 package snapshot
 
 import (
+	"sync"
+	"time"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/trie"
-	"sync"
 )
 
 type leaf struct {
@@ -32,7 +35,8 @@ type trieGeneratorFn func(in chan (leaf), out chan (common.Hash))
 
 // GenerateTrieRoot takes an account iterator and reproduces the root hash.
 func GenerateTrieRoot(it AccountIterator) common.Hash {
-	return generateTrieRoot(it, StackGenerate)
+	//return generateTrieRoot(it, StackGenerate)
+	return generateTrieRoot(it, StdGenerate)
 }
 
 func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Hash {
@@ -47,11 +51,24 @@ func generateTrieRoot(it AccountIterator, generatorFn trieGeneratorFn) common.Ha
 		wg.Done()
 	}()
 	// Feed leaves
+	start := time.Now()
+	logged := time.Now()
+	accounts := 0
 	for it.Next() {
-		in <- leaf{it.Hash(), it.Account()}
+		slimData := it.Account()
+		fullData := SlimToFull(slimData)
+		l := leaf{it.Hash(), fullData}
+		in <- l
+		if time.Since(logged) > 8*time.Second {
+			log.Info("Generating trie hash from snapshot",
+				"at", l.key, "accounts", accounts, "elapsed", time.Since(start))
+			logged = time.Now()
+		}
+		accounts++
 	}
 	close(in)
 	result := <-out
+	log.Info("Generated trie hash from snapshot", "accounts", accounts, "elapsed", time.Since(start))
 	wg.Wait()
 	return result
 }

core/state/snapshot/trie_generator_test.go

@@ -18,6 +18,8 @@ package snapshot
 
 import (
 	"encoding/binary"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/trie"
 	"math/rand"
 	"testing"
 
@@ -229,3 +231,24 @@ func BenchmarkTrieGeneration(b *testing.B) {
 		})
 	})
 }
+
+func TestStackVsStandard(t *testing.T) {
+	type kv struct {
+		key   string
+		value string
+	}
+	vals := []kv{
+		{key: "04f0860f1d82f4f0e61a03038cb0ffc08d15e22cb3d91d902c8acc32fa709b95", value: "f8440180a08e762c2b29fb1357d0794271a4dbe16167d8b28f1792ad9f78cad08206816127a010b37de11f39e0a372615c70e1d4d7c613937e8f61823d59be9bea62112e175c"},
+		{key: "04f0862f9177d381deeed0e6af3b0751f3cce6887746ba13cf41aa1c4dbf6591", value: "f8440180a014baf10561054a68fe522434b4d4c25e1b377e745bf1d676afa71bc891cacf9ba0debc58a981ca4f637e282ab5985d169a0237d03ea9336bc3434d9dce79e62ab3"},
+		{key: "04f0a6c0cb97e624bcb799f7d88717fe7fe4894877a8987a27d4792c36a2833e", value: "f8440180a0880595df1b6b3923e8036106cb641aae6b1249faa02d3217da8c556c0fff172ba06569f607421e3779a571977d84910e1177059946e0a064e487b1502e6a282623"},
+	}
+	stackT := trie.NewStackTrie()
+	stdT, _ := trie.New(common.Hash{}, trie.NewDatabase(memorydb.New()))
+	for _, kv := range vals {
+		stackT.TryUpdate(common.FromHex(kv.key), common.FromHex(kv.value))
+		stdT.TryUpdate(common.FromHex(kv.key), common.FromHex(kv.value))
+	}
+	if got, exp := stackT.Hash(), stdT.Hash(); got != exp {
+		t.Errorf("Hash mismatch, got %x, exp %x", got, exp)
+	}
+}