core/types: implement DerivableList interface for transactions and receipts (#21502)

This commit is contained in:
Daniel Liu 2025-03-25 17:57:10 +08:00 committed by Daniel Liu
parent dcea73039c
commit 3eb9f63513
10 changed files with 312 additions and 266 deletions

View file

@ -216,7 +216,7 @@ type storageblock struct {
// The values of TxHash, UncleHash, ReceiptHash and Bloom in header
// are ignored and set to values derived from the given txs, uncles
// and receipts.
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher Hasher) *Block {
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt, hasher TrieHasher) *Block {
b := &Block{header: CopyHeader(header), td: new(big.Int)}
// TODO: panic if len(txs) != len(receipts)

View file

@ -1,62 +0,0 @@
// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"bytes"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/rlp"
)
type DerivableList interface {
Len() int
GetRlp(i int) []byte
}
// Hasher is the tool used to calculate the hash of derivable list.
type Hasher interface {
Reset()
Update([]byte, []byte)
Hash() common.Hash
}
func DeriveSha(list DerivableList, hasher Hasher) common.Hash {
hasher.Reset()
keybuf := new(bytes.Buffer)
// StackTrie requires values to be inserted in increasing
// hash order, which is not the order that `list` provides
// hashes in. This insertion sequence ensures that the
// order is correct.
for i := 1; i < list.Len() && i <= 0x7f; i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
}
if list.Len() > 0 {
keybuf.Reset()
rlp.Encode(keybuf, uint(0))
hasher.Update(keybuf.Bytes(), list.GetRlp(0))
}
for i := 0x80; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
hasher.Update(keybuf.Bytes(), list.GetRlp(i))
}
return hasher.Hash()
}

View file

@ -31,7 +31,7 @@ var hasherPool = sync.Pool{
New: func() interface{} { return sha3.NewLegacyKeccak256() },
}
// deriveBufferPool holds temporary encoder buffers for DeriveSha and TX encoding.
// encodeBufferPool holds temporary encoder buffers for DeriveSha and TX encoding.
var encodeBufferPool = sync.Pool{
New: func() interface{} { return new(bytes.Buffer) },
}
@ -57,3 +57,57 @@ func prefixedRlpHash(prefix byte, x interface{}) (h common.Hash) {
sha.Read(h[:])
return h
}
// TrieHasher is the tool used to calculate the hash of derivable list.
// This is internal, do not use.
type TrieHasher interface {
Reset()
Update([]byte, []byte)
Hash() common.Hash
}
// DerivableList is the input to DeriveSha.
// It is implemented by the 'Transactions' and 'Receipts' types.
// This is internal, do not use these methods.
type DerivableList interface {
Len() int
EncodeIndex(int, *bytes.Buffer)
}
func encodeForDerive(list DerivableList, i int, buf *bytes.Buffer) []byte {
buf.Reset()
list.EncodeIndex(i, buf)
// It's really unfortunate that we need to do perform this copy.
// StackTrie holds onto the values until Hash is called, so the values
// written to it must not alias.
return common.CopyBytes(buf.Bytes())
}
// DeriveSha creates the tree hashes of transactions and receipts in a block header.
func DeriveSha(list DerivableList, hasher TrieHasher) common.Hash {
hasher.Reset()
valueBuf := encodeBufferPool.Get().(*bytes.Buffer)
defer encodeBufferPool.Put(valueBuf)
// StackTrie requires values to be inserted in increasing hash order, which is not the
// order that `list` provides hashes in. This insertion sequence ensures that the
// order is correct.
var indexBuf []byte
for i := 1; i < list.Len() && i <= 0x7f; i++ {
indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i))
value := encodeForDerive(list, i, valueBuf)
hasher.Update(indexBuf, value)
}
if list.Len() > 0 {
indexBuf = rlp.AppendUint64(indexBuf[:0], 0)
value := encodeForDerive(list, 0, valueBuf)
hasher.Update(indexBuf, value)
}
for i := 0x80; i < list.Len(); i++ {
indexBuf = rlp.AppendUint64(indexBuf[:0], uint64(i))
value := encodeForDerive(list, i, valueBuf)
hasher.Update(indexBuf, value)
}
return hasher.Hash()
}

212
core/types/hashing_test.go Normal file
View file

@ -0,0 +1,212 @@
package types_test
import (
"bytes"
"fmt"
"io"
"math/big"
mrand "math/rand"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/rlp"
"github.com/XinFinOrg/XDPoSChain/trie"
)
func TestDeriveSha(t *testing.T) {
txs, err := genTxs(0)
if err != nil {
t.Fatal(err)
}
for len(txs) < 1000 {
exp := types.DeriveSha(txs, new(trie.Trie))
got := types.DeriveSha(txs, trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
}
newTxs, err := genTxs(uint64(len(txs) + 1))
if err != nil {
t.Fatal(err)
}
txs = append(txs, newTxs...)
}
}
// TestEIP2718DeriveSha tests that the input to the DeriveSha function is correct.
func TestEIP2718DeriveSha(t *testing.T) {
for _, tc := range []struct {
rlpData string
exp string
}{
{
rlpData: "0xb8a701f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9",
exp: "01 01f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9\n80 01f8a486796f6c6f763380843b9aca008262d4948a8eafb1cf62bfbeb1741769dae1a9dd479961928080f838f7940000000000000000000000000000000000001337e1a0000000000000000000000000000000000000000000000000000000000000000080a0775101f92dcca278a56bfe4d613428624a1ebfc3cd9e0bcc1de80c41455b9021a06c9deac205afe7b124907d4ba54a9f46161498bd3990b90d175aac12c9a40ee9\n",
},
} {
d := &hashToHumanReadable{}
var t1, t2 types.Transaction
rlp.DecodeBytes(common.FromHex(tc.rlpData), &t1)
rlp.DecodeBytes(common.FromHex(tc.rlpData), &t2)
txs := types.Transactions{&t1, &t2}
types.DeriveSha(txs, d)
if tc.exp != string(d.data) {
t.Fatalf("Want\n%v\nhave:\n%v", tc.exp, string(d.data))
}
}
}
func BenchmarkDeriveSha200(b *testing.B) {
txs, err := genTxs(200)
if err != nil {
b.Fatal(err)
}
var exp common.Hash
var got common.Hash
b.Run("std_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, new(trie.Trie))
}
})
b.Run("stack_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
got = types.DeriveSha(txs, trie.NewStackTrie(nil))
}
})
if got != exp {
b.Errorf("got %x exp %x", got, exp)
}
}
func TestFuzzDeriveSha(t *testing.T) {
// increase this for longer runs -- it's set to quite low for travis
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), new(trie.Trie))
got := types.DeriveSha(newDummy(i), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
t.Fatalf("seed %d: got %x exp %x", seed, got, exp)
}
}
}
// TestDerivableList contains testcases found via fuzzing
func TestDerivableList(t *testing.T) {
type tcase []string
tcs := []tcase{
{
"0xc041",
},
{
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
},
{
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
"0x6cd850eca0a7ac46bb1748d7b9cb88aa3bd21c57d852c28198ad8fa422c4595032e88a4494b4778b36b944fe47a52b8c5cd312910139dfcb4147ab8e972cc456bcb063f25dd78f54c4d34679e03142c42c662af52947d45bdb6e555751334ace76a5080ab5a0256a1d259855dfc5c0b8023b25befbb13fd3684f9f755cbd3d63544c78ee2001452dd54633a7593ade0b183891a0a4e9c7844e1254005fbe592b1b89149a502c24b6e1dca44c158aebedf01beae9c30cabe16a",
"0x14abd5c47c0be87b0454596baad2",
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
},
}
for i, tc := range tcs[1:] {
exp := types.DeriveSha(flatList(tc), new(trie.Trie))
got := types.DeriveSha(flatList(tc), trie.NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)
}
}
}
func genTxs(num uint64) (types.Transactions, error) {
key, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
if err != nil {
return nil, err
}
var addr = crypto.PubkeyToAddress(key.PublicKey)
newTx := func(i uint64) (*types.Transaction, error) {
signer := types.NewEIP155Signer(big.NewInt(18))
utx := types.NewTransaction(i, addr, new(big.Int), 0, new(big.Int).SetUint64(10000000), nil)
tx, err := types.SignTx(utx, signer, key)
return tx, err
}
var txs types.Transactions
for i := uint64(0); i < num; i++ {
tx, err := newTx(i)
if err != nil {
return nil, err
}
txs = append(txs, tx)
}
return txs, nil
}
type dummyDerivableList struct {
len int
seed int
}
func newDummy(seed int) *dummyDerivableList {
d := &dummyDerivableList{}
src := mrand.NewSource(int64(seed))
// don't use lists longer than 4K items
d.len = int(src.Int63() & 0x0FFF)
d.seed = seed
return d
}
func (d *dummyDerivableList) Len() int {
return d.len
}
func (d *dummyDerivableList) EncodeIndex(i int, w *bytes.Buffer) {
src := mrand.NewSource(int64(d.seed + i))
// max item size 256, at least 1 byte per item
size := 1 + src.Int63()&0x00FF
io.CopyN(w, mrand.New(src), size)
}
func printList(l types.DerivableList) {
fmt.Printf("list length: %d\n", l.Len())
fmt.Printf("{\n")
for i := 0; i < l.Len(); i++ {
var buf bytes.Buffer
l.EncodeIndex(i, &buf)
fmt.Printf("\"0x%x\",\n", buf.Bytes())
}
fmt.Printf("},\n")
}
type flatList []string
func (f flatList) Len() int {
return len(f)
}
func (f flatList) EncodeIndex(i int, w *bytes.Buffer) {
w.Write(hexutil.MustDecode(f[i]))
}
type hashToHumanReadable struct {
data []byte
}
func (d *hashToHumanReadable) Reset() {
d.data = make([]byte, 0)
}
func (d *hashToHumanReadable) Update(i []byte, i2 []byte) {
l := fmt.Sprintf("%x %x\n", i, i2)
d.data = append(d.data, []byte(l)...)
}
func (d *hashToHumanReadable) Hash() common.Hash {
return common.Hash{}
}

View file

@ -17,6 +17,7 @@
package types
import (
"bytes"
"container/heap"
"errors"
"io"
@ -295,13 +296,9 @@ type LendingTransactions []*LendingTransaction
// Len returns the length of s.
func (s LendingTransactions) Len() int { return len(s) }
// Swap swaps the i'th and the j'th element in s.
func (s LendingTransactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// GetRlp implements Rlpable and returns the i'th element of s in rlp.
func (s LendingTransactions) GetRlp(i int) []byte {
enc, _ := rlp.EncodeToBytes(s[i])
return enc
// EncodeIndex encodes the i'th element of s to w.
func (s LendingTransactions) EncodeIndex(i int, w *bytes.Buffer) {
rlp.Encode(w, s[i])
}
// LendingTxDifference returns a new set t which is the difference between a to b.

View file

@ -17,6 +17,7 @@
package types
import (
"bytes"
"container/heap"
"errors"
"io"
@ -234,13 +235,9 @@ type OrderTransactions []*OrderTransaction
// Len returns the length of s.
func (s OrderTransactions) Len() int { return len(s) }
// Swap swaps the i'th and the j'th element in s.
func (s OrderTransactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// GetRlp implements Rlpable and returns the i'th element of s in rlp.
func (s OrderTransactions) GetRlp(i int) []byte {
enc, _ := rlp.EncodeToBytes(s[i])
return enc
// EncodeIndex encodes the i'th element of s to w.
func (s OrderTransactions) EncodeIndex(i int, w *bytes.Buffer) {
rlp.Encode(w, s[i])
}
// OrderTxDifference returns a new set t which is the difference between a to b.

View file

@ -379,13 +379,23 @@ type Receipts []*Receipt
// Len returns the number of receipts in this list.
func (rs Receipts) Len() int { return len(rs) }
// GetRlp returns the RLP encoding of one receipt from the list.
func (rs Receipts) GetRlp(i int) []byte {
bytes, err := rlp.EncodeToBytes(rs[i])
if err != nil {
panic(err)
// EncodeIndex encodes the i'th receipt to w.
func (rs Receipts) EncodeIndex(i int, w *bytes.Buffer) {
r := rs[i]
data := &receiptRLP{r.statusEncoding(), r.CumulativeGasUsed, r.Bloom, r.Logs}
if r.Type == LegacyTxType {
rlp.Encode(w, data)
return
}
w.WriteByte(r.Type)
switch r.Type {
case AccessListTxType, DynamicFeeTxType:
rlp.Encode(w, data)
default:
// For unsupported types, write nothing. Since this is for
// DeriveSha, the error will be caught matching the derived hash
// to the block.
}
return bytes
}
// DeriveFields fills the receipts with their computed fields based on consensus

View file

@ -632,13 +632,16 @@ type Transactions []*Transaction
// Len returns the length of s.
func (s Transactions) Len() int { return len(s) }
// Swap swaps the i'th and the j'th element in s.
func (s Transactions) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
// GetRlp implements Rlpable and returns the i'th element of s in rlp.
func (s Transactions) GetRlp(i int) []byte {
enc, _ := rlp.EncodeToBytes(s[i])
return enc
// EncodeIndex encodes the i'th transaction to w. Note that this does not check for errors
// because we assume that *Transaction will only ever contain valid txs that were either
// constructed by decoding or via public API in this package.
func (s Transactions) EncodeIndex(i int, w *bytes.Buffer) {
tx := s[i]
if tx.Type() == LegacyTxType {
rlp.Encode(w, tx.inner)
} else {
tx.encodeTyped(w)
}
}
// TxDifference returns a new set t which is the difference between a to b.

View file

@ -3,6 +3,7 @@ package ethapi
import (
"bytes"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/rlp"
@ -28,12 +29,16 @@ func (n *proofPairList) Delete(key []byte) error {
// modified from core/types/derive_sha.go
func deriveTrie(list types.DerivableList) *trie.Trie {
keybuf := new(bytes.Buffer)
buf := new(bytes.Buffer)
trie := new(trie.Trie)
for i := 0; i < list.Len(); i++ {
keybuf.Reset()
rlp.Encode(keybuf, uint(i))
trie.Update(keybuf.Bytes(), list.GetRlp(i))
for i := range list.Len() {
buf.Reset()
rlp.Encode(buf, uint(i))
key := common.CopyBytes(buf.Bytes())
buf.Reset()
list.EncodeIndex(i, buf)
value := common.CopyBytes(buf.Bytes())
trie.Update(key, value)
}
return trie
}

View file

@ -1,16 +1,9 @@
package trie
import (
"bytes"
"fmt"
"math/big"
mrand "math/rand"
"testing"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
)
@ -78,169 +71,6 @@ func TestValLength56(t *testing.T) {
}
}
func genTxs(num uint64) (types.Transactions, error) {
key, err := crypto.HexToECDSA("deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
if err != nil {
return nil, err
}
var addr = crypto.PubkeyToAddress(key.PublicKey)
newTx := func(i uint64) (*types.Transaction, error) {
signer := types.NewEIP155Signer(big.NewInt(18))
tx, err := types.SignTx(types.NewTransaction(i, addr, new(big.Int), 0, new(big.Int).SetUint64(10000000), nil), signer, key)
return tx, err
}
var txs types.Transactions
for i := uint64(0); i < num; i++ {
tx, err := newTx(i)
if err != nil {
return nil, err
}
txs = append(txs, tx)
}
return txs, nil
}
func TestDeriveSha(t *testing.T) {
txs, err := genTxs(0)
if err != nil {
t.Fatal(err)
}
for len(txs) < 1000 {
exp := types.DeriveSha(txs, newEmpty())
got := types.DeriveSha(txs, NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("%d txs: got %x exp %x", len(txs), got, exp)
}
newTxs, err := genTxs(uint64(len(txs) + 1))
if err != nil {
t.Fatal(err)
}
txs = append(txs, newTxs...)
}
}
func BenchmarkDeriveSha200(b *testing.B) {
txs, err := genTxs(200)
if err != nil {
b.Fatal(err)
}
var exp common.Hash
var got common.Hash
b.Run("std_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
exp = types.DeriveSha(txs, newEmpty())
}
})
b.Run("stack_trie", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
got = types.DeriveSha(txs, NewStackTrie(nil))
}
})
if got != exp {
b.Errorf("got %x exp %x", got, exp)
}
}
type dummyDerivableList struct {
len int
seed int
}
func newDummy(seed int) *dummyDerivableList {
d := &dummyDerivableList{}
src := mrand.NewSource(int64(seed))
// don't use lists longer than 4K items
d.len = int(src.Int63() & 0x0FFF)
d.seed = seed
return d
}
func (d *dummyDerivableList) Len() int {
return d.len
}
func (d *dummyDerivableList) GetRlp(i int) []byte {
src := mrand.NewSource(int64(d.seed + i))
// max item size 256, at least 1 byte per item
size := 1 + src.Int63()&0x00FF
data := make([]byte, size)
_, err := mrand.New(src).Read(data)
if err != nil {
panic(err)
}
return data
}
func printList(l types.DerivableList) {
fmt.Printf("list length: %d\n", l.Len())
fmt.Printf("{\n")
for i := 0; i < l.Len(); i++ {
v := l.GetRlp(i)
fmt.Printf("\"0x%x\",\n", v)
}
fmt.Printf("},\n")
}
func TestFuzzDeriveSha(t *testing.T) {
// increase this for longer runs -- it's set to quite low for travis
rndSeed := mrand.Int()
for i := 0; i < 10; i++ {
seed := rndSeed + i
exp := types.DeriveSha(newDummy(i), newEmpty())
got := types.DeriveSha(newDummy(i), NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
printList(newDummy(seed))
t.Fatalf("seed %d: got %x exp %x", seed, got, exp)
}
}
}
type flatList struct {
rlpvals []string
}
func newFlatList(rlpvals []string) *flatList {
return &flatList{rlpvals}
}
func (f *flatList) Len() int {
return len(f.rlpvals)
}
func (f *flatList) GetRlp(i int) []byte {
return hexutil.MustDecode(f.rlpvals[i])
}
// TestDerivableList contains testcases found via fuzzing
func TestDerivableList(t *testing.T) {
type tcase []string
tcs := []tcase{
{
"0xc041",
},
{
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
"0xf04cf757812428b0763112efb33b6f4fad7deb445e",
},
{
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
"0x6cd850eca0a7ac46bb1748d7b9cb88aa3bd21c57d852c28198ad8fa422c4595032e88a4494b4778b36b944fe47a52b8c5cd312910139dfcb4147ab8e972cc456bcb063f25dd78f54c4d34679e03142c42c662af52947d45bdb6e555751334ace76a5080ab5a0256a1d259855dfc5c0b8023b25befbb13fd3684f9f755cbd3d63544c78ee2001452dd54633a7593ade0b183891a0a4e9c7844e1254005fbe592b1b89149a502c24b6e1dca44c158aebedf01beae9c30cabe16a",
"0x14abd5c47c0be87b0454596baad2",
"0xca410605310cdc3bb8d4977ae4f0143df54a724ed873457e2272f39d66e0460e971d9d",
},
}
for i, tc := range tcs[1:] {
exp := types.DeriveSha(newFlatList(tc), newEmpty())
got := types.DeriveSha(newFlatList(tc), NewStackTrie(nil))
if !bytes.Equal(got[:], exp[:]) {
t.Fatalf("case %d: got %x exp %x", i, got, exp)
}
}
}
// TestUpdateSmallNodes tests a case where the leaves are small (both key and value),
// which causes a lot of node-within-node. This case was found via fuzzing.
func TestUpdateSmallNodes(t *testing.T) {