core/state/snapshot: faster slim-to-hash method

core/state/snapshot/account.go

@@ -21,7 +21,9 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"golang.org/x/crypto/sha3"
 )
 
 // Account is a slim version of a state.Account, where the root and code hash
@@ -68,3 +70,64 @@ func SlimToFull(data []byte) []byte {
 	}
 	return fullData
 }
+
+// conversionAccount is used for converting between full and slim format. When
+// doing this, we can consider 'balance' as a byte array, as it has already
+// been converted from big.Int into an rlp-byteslice.
+type conversionAccount struct {
+	Nonce    uint64
+	Balance  []byte
+	Root     []byte
+	CodeHash []byte
+}
+
+type converter struct {
+	tmpAcc *conversionAccount
+	sha3   crypto.KeccakState
+	stream rlp.Stream
+}
+
+func newConverter() *converter {
+	return &converter{
+		tmpAcc: &conversionAccount{},
+		sha3:   sha3.NewLegacyKeccak256().(crypto.KeccakState),
+	}
+}
+
+func (c *converter) SlimToHash(data []byte) common.Hash {
+	var (
+		result common.Hash
+		tmp    = c.tmpAcc
+		sha3   = c.sha3
+	)
+	c.stream.Reset(bytes.NewReader(data), 0)
+	c.stream.Decode(c.tmpAcc)
+	if len(tmp.Root) == 0 {
+		tmp.Root = emptyRoot[:]
+	}
+	if len(tmp.CodeHash) == 0 {
+		tmp.CodeHash = emptyCode[:]
+	}
+	sha3.Reset()
+	_ = rlp.Encode(sha3, tmp)
+	sha3.Read(result[:])
+	return result
+}
+
+// SlimToHash produces a hash of a main account trie, where the input is the
+// 'slim' version
+func SlimToHash(data []byte, sha3 crypto.KeccakState) common.Hash {
+	tmp := &conversionAccount{}
+	var result common.Hash
+	rlp.DecodeBytes(data, tmp)
+	if len(tmp.Root) == 0 {
+		tmp.Root = emptyRoot[:]
+	}
+	if len(tmp.CodeHash) == 0 {
+		tmp.CodeHash = emptyCode[:]
+	}
+	sha3.Reset()
+	_ = rlp.Encode(sha3, tmp)
+	sha3.Read(result[:])
+	return result
+}

core/state/snapshot/trie_generator_test.go

@@ -19,14 +19,17 @@ package snapshot
 import (
 	"bytes"
 	"encoding/binary"
-	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"math/big"
-	"github.com/ethereum/go-ethereum/trie"
 	"math/rand"
 	"testing"
 
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/trie"
+	"golang.org/x/crypto/sha3"
 )
 
 func TestTrieGeneration(t *testing.T) {
@@ -291,3 +294,75 @@ func TestReStackTrieLeafInsert(t *testing.T) {
 		t.Fatalf("Invalid hash, expected %s got %s", common.ToHex(ref.Hash().Bytes()), common.ToHex(root.Hash().Bytes()))
 	}
 }
+
+func TestSlimToFullHash(t *testing.T) {
+	rand.Seed(1881)
+	var slimAccounts [][]byte
+	for i := 0; i < 10000; i++ {
+		slimData := AccountRLP(rand.Uint64(),
+			big.NewInt(0).SetUint64(rand.Uint64()),
+			randomHash(),
+			randomHash().Bytes())
+		slimAccounts = append(slimAccounts, slimData)
+	}
+	hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState)
+	for _, slimData := range slimAccounts {
+		// reference
+		expanded := SlimToFull(slimData)
+		exp := crypto.Keccak256Hash(expanded)
+		got := SlimToHash(slimData, hasher)
+		if got != exp {
+			t.Fatalf("got %x exp %x \ndata: %x", got, exp, slimData)
+		}
+	}
+}
+
+func BenchmarkSlimToFullHash(b *testing.B) {
+	rand.Seed(1881)
+	var slimAccounts [][]byte
+	for i := 0; i < 10000; i++ {
+		slimData := AccountRLP(rand.Uint64(),
+			big.NewInt(0).SetUint64(rand.Uint64()),
+			randomHash(),
+			randomHash().Bytes())
+		slimAccounts = append(slimAccounts, slimData)
+	}
+	b.ResetTimer()
+	var exp, got common.Hash
+	b.Run("naive-10K", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			for _, slimData := range slimAccounts {
+				// reference
+				expanded := SlimToFull(slimData)
+				exp = crypto.Keccak256Hash(expanded)
+			}
+		}
+	})
+	hasher := sha3.NewLegacyKeccak256().(crypto.KeccakState)
+
+	b.Run("directToHash-10K", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			for _, slimData := range slimAccounts {
+				got = SlimToHash(slimData, hasher)
+			}
+		}
+	})
+	if got != exp {
+		b.Fatalf("got %x exp %x", got, exp)
+	}
+	c := newConverter()
+	b.Run("directToHashBuf-10K", func(b *testing.B) {
+		b.ReportAllocs()
+		for i := 0; i < b.N; i++ {
+			for _, slimData := range slimAccounts {
+				got = c.SlimToHash(slimData)
+			}
+		}
+	})
+	if got != exp {
+		b.Fatalf("got %x exp %x", got, exp)
+	}
+
+}

crypto/crypto.go

@@ -23,6 +23,7 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"hash"
 	"io"
 	"io/ioutil"
 	"math/big"
@@ -79,6 +80,25 @@ func Keccak512(data ...[]byte) []byte {
 	return d.Sum(nil)
 }
 
+// KeccakState wraps sha3.state. In addition to the usual hash methods, it also supports
+// Read to get a variable amount of data from the hash state. Read is faster than Sum
+// because it doesn't copy the internal state, but also modifies the internal state.
+type KeccakState interface {
+	hash.Hash
+	Read([]byte) (int, error)
+}
+
+type SliceBuffer []byte
+
+func (b *SliceBuffer) Write(data []byte) (n int, err error) {
+	*b = append(*b, data...)
+	return len(data), nil
+}
+
+func (b *SliceBuffer) Reset() {
+	*b = (*b)[:0]
+}
+
 // CreateAddress creates an ethereum address given the bytes and the nonce
 func CreateAddress(b common.Address, nonce uint64) common.Address {
 	data, _ := rlp.EncodeToBytes([]interface{}{b, nonce})

trie/committer.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/crypto/sha3"
 )
@@ -45,8 +46,8 @@ type leaf struct {
 // By 'some level' of parallelism, it's still the case that all leaves will be
 // processed sequentially - onleaf will never be called in parallel or out of order.
 type committer struct {
-	tmp sliceBuffer
+	tmp crypto.SliceBuffer
-	sha keccakState
+	sha crypto.KeccakState
 
 	onleaf LeafCallback
 	leafCh chan *leaf
@@ -56,8 +57,8 @@ type committer struct {
 var committerPool = sync.Pool{
 	New: func() interface{} {
 		return &committer{
-			tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
+			tmp: make(crypto.SliceBuffer, 0, 550), // cap is as large as a full fullNode.
-			sha: sha3.NewLegacyKeccak256().(keccakState),
+			sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
 		}
 	},
 }

trie/hasher.go

@@ -17,37 +17,18 @@
 package trie
 
 import (
-	"hash"
 	"sync"
 
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/crypto/sha3"
 )
 
-// keccakState wraps sha3.state. In addition to the usual hash methods, it also supports
-// Read to get a variable amount of data from the hash state. Read is faster than Sum
-// because it doesn't copy the internal state, but also modifies the internal state.
-type keccakState interface {
-	hash.Hash
-	Read([]byte) (int, error)
-}
-
-type sliceBuffer []byte
-
-func (b *sliceBuffer) Write(data []byte) (n int, err error) {
-	*b = append(*b, data...)
-	return len(data), nil
-}
-
-func (b *sliceBuffer) Reset() {
-	*b = (*b)[:0]
-}
-
 // hasher is a type used for the trie Hash operation. A hasher has some
 // internal preallocated temp space
 type hasher struct {
-	sha      keccakState
+	sha      crypto.KeccakState
-	tmp      sliceBuffer
+	tmp      crypto.SliceBuffer
 	parallel bool // Whether to use paralallel threads when hashing
 }
 
@@ -55,8 +36,8 @@ type hasher struct {
 var hasherPool = sync.Pool{
 	New: func() interface{} {
 		return &hasher{
-			tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode.
+			tmp: make(crypto.SliceBuffer, 0, 550), // cap is as large as a full fullNode.
-			sha: sha3.NewLegacyKeccak256().(keccakState),
+			sha: sha3.NewLegacyKeccak256().(crypto.KeccakState),
 		}
 	},
 }