From 6dc45cf878f9d17c4e73425b7cf4adafed6d161b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Mar 2020 12:42:57 +0100 Subject: [PATCH] core/state/snapshot: faster slim-to-hash method --- core/state/snapshot/account.go | 63 +++++++++++++++++ core/state/snapshot/trie_generator_test.go | 79 +++++++++++++++++++++- crypto/crypto.go | 20 ++++++ trie/committer.go | 9 +-- trie/hasher.go | 29 ++------ 5 files changed, 170 insertions(+), 30 deletions(-) diff --git a/core/state/snapshot/account.go b/core/state/snapshot/account.go index 496e1a39df..4884a58a7a 100644 --- a/core/state/snapshot/account.go +++ b/core/state/snapshot/account.go @@ -21,7 +21,9 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" ) // Account is a slim version of a state.Account, where the root and code hash @@ -68,3 +70,64 @@ func SlimToFull(data []byte) []byte { } return fullData } + +// conversionAccount is used for converting between full and slim format. When +// doing this, we can consider 'balance' as a byte array, as it has already +// been converted from big.Int into an rlp-byteslice. +type conversionAccount struct { + Nonce uint64 + Balance []byte + Root []byte + CodeHash []byte +} + +type converter struct { + tmpAcc *conversionAccount + sha3 crypto.KeccakState + stream rlp.Stream +} + +func newConverter() *converter { + return &converter{ + tmpAcc: &conversionAccount{}, + sha3: sha3.NewLegacyKeccak256().(crypto.KeccakState), + } +} + +func (c *converter) SlimToHash(data []byte) common.Hash { + var ( + result common.Hash + tmp = c.tmpAcc + sha3 = c.sha3 + ) + c.stream.Reset(bytes.NewReader(data), 0) + c.stream.Decode(c.tmpAcc) + if len(tmp.Root) == 0 { + tmp.Root = emptyRoot[:] + } + if len(tmp.CodeHash) == 0 { + tmp.CodeHash = emptyCode[:] + } + sha3.Reset() + _ = rlp.Encode(sha3, tmp) + sha3.Read(result[:]) + return result +} + +// SlimToHash produces a hash of a main account trie, where the input is the +// 'slim' version +func SlimToHash(data []byte, sha3 crypto.KeccakState) common.Hash { + tmp := &conversionAccount{} + var result common.Hash + rlp.DecodeBytes(data, tmp) + if len(tmp.Root) == 0 { + tmp.Root = emptyRoot[:] + } + if len(tmp.CodeHash) == 0 { + tmp.CodeHash = emptyCode[:] + } + sha3.Reset() + _ = rlp.Encode(sha3, tmp) + sha3.Read(result[:]) + return result +} diff --git a/core/state/snapshot/trie_generator_test.go b/core/state/snapshot/trie_generator_test.go index 16f7ad4772..c79ea73af0 100644 --- a/core/state/snapshot/trie_generator_test.go +++ b/core/state/snapshot/trie_generator_test.go @@ -19,14 +19,17 @@ package snapshot import ( "bytes" "encoding/binary" - "github.com/ethereum/go-ethereum/ethdb/memorydb" - "github.com/ethereum/go-ethereum/trie" + "math/big" "math/rand" "testing" "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/trie" + "golang.org/x/crypto/sha3" ) func TestTrieGeneration(t *testing.T) { @@ -291,3 +294,75 @@ func TestReStackTrieLeafInsert(t *testing.T) { t.Fatalf("Invalid hash, expected %s got %s", common.ToHex(ref.Hash().Bytes()), common.ToHex(root.Hash().Bytes())) } } + +func TestSlimToFullHash(t *testing.T) { + rand.Seed(1881) + var slimAccounts [][]byte + for i := 0; i < 10000; i++ { + slimData := AccountRLP(rand.Uint64(), + big.NewInt(0).SetUint64(rand.Uint64()), + randomHash(), + randomHash().Bytes()) + slimAccounts = append(slimAccounts, slimData) + } + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + for _, slimData := range slimAccounts { + // reference + expanded := SlimToFull(slimData) + exp := crypto.Keccak256Hash(expanded) + got := SlimToHash(slimData, hasher) + if got != exp { + t.Fatalf("got %x exp %x \ndata: %x", got, exp, slimData) + } + } +} + +func BenchmarkSlimToFullHash(b *testing.B) { + rand.Seed(1881) + var slimAccounts [][]byte + for i := 0; i < 10000; i++ { + slimData := AccountRLP(rand.Uint64(), + big.NewInt(0).SetUint64(rand.Uint64()), + randomHash(), + randomHash().Bytes()) + slimAccounts = append(slimAccounts, slimData) + } + b.ResetTimer() + var exp, got common.Hash + b.Run("naive-10K", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, slimData := range slimAccounts { + // reference + expanded := SlimToFull(slimData) + exp = crypto.Keccak256Hash(expanded) + } + } + }) + hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState) + + b.Run("directToHash-10K", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, slimData := range slimAccounts { + got = SlimToHash(slimData, hasher) + } + } + }) + if got != exp { + b.Fatalf("got %x exp %x", got, exp) + } + c := newConverter() + b.Run("directToHashBuf-10K", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, slimData := range slimAccounts { + got = c.SlimToHash(slimData) + } + } + }) + if got != exp { + b.Fatalf("got %x exp %x", got, exp) + } + +} diff --git a/crypto/crypto.go b/crypto/crypto.go index 2869b4c191..03725651b2 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -23,6 +23,7 @@ import ( "encoding/hex" "errors" "fmt" + "hash" "io" "io/ioutil" "math/big" @@ -79,6 +80,25 @@ func Keccak512(data ...[]byte) []byte { return d.Sum(nil) } +// KeccakState wraps sha3.state. In addition to the usual hash methods, it also supports +// Read to get a variable amount of data from the hash state. Read is faster than Sum +// because it doesn't copy the internal state, but also modifies the internal state. +type KeccakState interface { + hash.Hash + Read([]byte) (int, error) +} + +type SliceBuffer []byte + +func (b *SliceBuffer) Write(data []byte) (n int, err error) { + *b = append(*b, data...) + return len(data), nil +} + +func (b *SliceBuffer) Reset() { + *b = (*b)[:0] +} + // CreateAddress creates an ethereum address given the bytes and the nonce func CreateAddress(b common.Address, nonce uint64) common.Address { data, _ := rlp.EncodeToBytes([]interface{}{b, nonce}) diff --git a/trie/committer.go b/trie/committer.go index eacefdff11..04bc00123c 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) @@ -45,8 +46,8 @@ type leaf struct { // By 'some level' of parallelism, it's still the case that all leaves will be // processed sequentially - onleaf will never be called in parallel or out of order. type committer struct { - tmp sliceBuffer - sha keccakState + tmp crypto.SliceBuffer + sha crypto.KeccakState onleaf LeafCallback leafCh chan *leaf @@ -56,8 +57,8 @@ type committer struct { var committerPool = sync.Pool{ New: func() interface{} { return &committer{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. - sha: sha3.NewLegacyKeccak256().(keccakState), + tmp: make(crypto.SliceBuffer, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), } }, } diff --git a/trie/hasher.go b/trie/hasher.go index 8e8eec9f61..0333f38805 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -17,37 +17,18 @@ package trie import ( - "hash" "sync" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) -// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports -// Read to get a variable amount of data from the hash state. Read is faster than Sum -// because it doesn't copy the internal state, but also modifies the internal state. -type keccakState interface { - hash.Hash - Read([]byte) (int, error) -} - -type sliceBuffer []byte - -func (b *sliceBuffer) Write(data []byte) (n int, err error) { - *b = append(*b, data...) - return len(data), nil -} - -func (b *sliceBuffer) Reset() { - *b = (*b)[:0] -} - // hasher is a type used for the trie Hash operation. A hasher has some // internal preallocated temp space type hasher struct { - sha keccakState - tmp sliceBuffer + sha crypto.KeccakState + tmp crypto.SliceBuffer parallel bool // Whether to use paralallel threads when hashing } @@ -55,8 +36,8 @@ type hasher struct { var hasherPool = sync.Pool{ New: func() interface{} { return &hasher{ - tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. - sha: sha3.NewLegacyKeccak256().(keccakState), + tmp: make(crypto.SliceBuffer, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewLegacyKeccak256().(crypto.KeccakState), } }, }