forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-27 07:06:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor: adapt to standalone zktrie module (#165)

Browse source 
* adapt zktrie: zktrie part

* adapt unittests

* pass tests

* refactor for new zktrie module

* run go imports
This commit is contained in:
Ho 2022-10-17 14:32:42 +08:00 committed by GitHub
parent b3a7c9b691
commit e6500918a0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 234 additions and 1956 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
core/state/statedb.go
View file

@ -24,6 +24,8 @@ import (
"sort"
"time"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/rawdb"
"github.com/scroll-tech/go-ethereum/core/state/snapshot"
@ -316,18 +318,14 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash {
// GetProof returns the Merkle proof for a given account.
func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) {
if s.IsZktrie() {
var proof proofList
err := s.trie.Prove(addr.Bytes32(), 0, &proof)
return proof, err
addr_s, _ := zkt.ToSecureKeyBytes(addr.Bytes())
return s.GetProofByHash(common.BytesToHash(addr_s.Bytes()))
}
return s.GetProofByHash(crypto.Keccak256Hash(addr.Bytes()))
}
// GetProofByHash returns the Merkle proof for a given account.
func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) {
if s.IsZktrie() {
panic("unimplemented")
}
var proof proofList
err := s.trie.Prove(addrHash[:], 0, &proof)
return proof, err
@ -366,7 +364,8 @@ func (s *StateDB) GetStorageTrieProof(a common.Address, key common.Hash) ([][]by
var proof proofList
if s.IsZktrie() {
err = trie.Prove(key.Bytes(), 0, &proof)
key_s, _ := zkt.ToSecureKeyBytes(key.Bytes())
err = trie.Prove(key_s.Bytes(), 0, &proof)
} else {
err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
}
@ -382,7 +381,8 @@ func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte,
}
var err error
if s.IsZktrie() {
err = trie.Prove(key.Bytes(), 0, &proof)
key_s, _ := zkt.ToSecureKeyBytes(key.Bytes())
err = trie.Prove(key_s.Bytes(), 0, &proof)
} else {
err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof)
}

8
core/types/state_account_marshalling.go
View file

@ -24,8 +24,9 @@ import (
"github.com/iden3/go-iden3-crypto/poseidon"
"github.com/iden3/go-iden3-crypto/utils"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
)
var (
@ -56,10 +57,7 @@ func (s *StateAccount) Hash() (*big.Int, error) {
return nil, err
}
rootHash, err := zkt.NewHashFromBytes(s.Root.Bytes())
if err != nil {
return nil, err
}
rootHash := zkt.NewHashFromBytes(s.Root.Bytes())
hash3, err := poseidon.Hash([]*big.Int{hash2, rootHash.BigInt()})
if err != nil {
return nil, err

3
core/types/state_account_marshalling_test.go
View file

@ -4,8 +4,9 @@ import (
"math/big"
"testing"
zktrie "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types/zktrie"
)
func TestAccountMarshalling(t *testing.T) {

143
core/types/zktrie/README.md
View file

@ -1,143 +0,0 @@
# Type for zktrie
## Data Format in stateDb
All data node being stored via stateDb are encoded by following syntax:
``` EBNF
node = magic string | node data ;
magic string = "THIS IS SOME MAGIC BYTES FOR SMT m1rRXgP2xpDI" ;
node data = middle node | leaf node | empty node ;
empty node = '0x2' ;
middle node = '0x0', left hash, right hash ;
field = 32 * hex char ;
left hash = field ;
right hash = field ;
leaf node = node key , value len , compress flag , <value len> * value field, key preimage ;
node key = field ;
compress flag = 3 * byte ;
value len = byte ;
value field = field | compressed field, compressed field ;
compressed field = 16 * hex char ;
key preimage = '0x0' | preimage bytes ;
preimage bytes = len, <len> * byte ;
len = byte ;
```
A `field` is an element in prime field of BN256 represented by **big endian** integer and contained in fixed length (32) bytes;
A `compressed field` is a field represented by **big endian** integer which could be contained in 16 bytes;
For the total `value len` items of `value field` (maximum 255), the first 24 `value field`s can be recorded as `field` or 2x `compressed field` (i.e. a byte32). The corresonpdoing bit in `compress flag` is set to 1 if it was recorded as byte32, or 0 for a field.
## Key scheme
The key of data node is obtained from one or more poseidon hash calculation: `poseidon := (field, field) => field`.
For middle node:
```
key = poseidon(<left hash>, <right hash>)
```
For leaf node:
```
key = poseidon(<pre key>, <value hash>)
pre key = poseidon(field(1), <node key>)
value hash = poseidon(<leaf element>, <leaf element>) | poseidon(<value hash>, <value hash>)
leaf element = <value field as field> | poseidon(<compressed field as field>, <compressed field as field>) | field(0)
```
That is, to calculate the key of a leaf node:
1. In the sequence of `value field`s, take which is recorded as 'compressed' and calculate the 2x `compressed field` for its poseidon hash, replace the corresponding `value field` ad-hoc in the sequence;
2. Consider the sequence from 1 as the leafs of a binary merkle tree (append a 0 field for odd leafs) and calculate its root by poseidon hash;
For empty node:
```
key = field(0)
```
## Account data
Each account data is saved in one leaf node of account zktrie as 4 `value field`s:
1. Nonce as `field`
2. Balance as `field`
3. CodeHash as `compressed field` (byte32)
4. Storage root as `field`
The key for an account data is calculated from the 20-bit account address as following:
```
32-byte-zero-end-padding-addr := address, 16 * bytes (0)
key = poseidon(<first 16 byte of 32-byte-zero-end-padding-addr as field>, <last 16 byte of 32-byte-zero-end-padding-addr as field>)
```
## Data examples
### A leaf node in account trie:
> 0x017f9d3bbc51d12566ecc6049ca6bf76e32828c22b197405f63a833b566fe7da0a040400000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000029b74e075daad9f17eb39cd893c2dd32f52ecd99084d63964842defd00ebcbe208a2f471d50e56ac5000ab9e82f871e36b5a636b19bd02f70aa666a3bd03142f00
Can be decompose to:
+ `0x01`: node type prefix for leaf node
+ `7f9d3bbc51d12566ecc6049ca6bf76e32828c22b197405f63a833b566fe7da0a`: node key as field
+ `04`: value len (4 value fields)
+ `040000`: compress flag, a 24 bit array, indicating the third field is compressed
+ `0000000000000000000000000000000000000000000000000000000000000001`: value field 0 (nonce)
+ `0000000000000000000000000000000000000000000000000000000000000000`: value field 1 (balance)
+ `29b74e075daad9f17eb39cd893c2dd32f52ecd99084d63964842defd00ebcbe2`: value field 2 (codeHash, as byte32)
+ `08a2f471d50e56ac5000ab9e82f871e36b5a636b19bd02f70aa666a3bd03142f`: value field 3 (storage root)
+ `00`: key preimage is not avaliable
The key calculation for this node is:
```
arr = [<value field 0>, <value field 1>, <value field 2>, <value field 3>]
hash_pre = poseidon(<first 16 byte for value field 2>, <last 16 byte for value field 2>)
arr[2] = hash_pre
layer1 = [poseidon(arr[0], arr[1]), poseidon(arr[2], arr[3])]
key = poseidon(layer1[0], layer1[1])
```
Notice all field and compressed field are represented as **big endian** integer.
### A middle node in account trie:
> 0x00000000000000000000000000000000000000000000000000000000000000000004470b58d80eeb26da85b2c2db5c254900656fb459c07729f556ff02534ab32a
Notice the left child of this node is an empty node (so its key is field(0))

45
core/types/zktrie/byte32.go
View file

@ -1,45 +0,0 @@
package zktrie
import (
"fmt"
"math/big"
"github.com/iden3/go-iden3-crypto/poseidon"
)
type Byte32 [32]byte
func (b *Byte32) Hash() (*big.Int, error) {
first16 := new(big.Int).SetBytes(b[0:16])
last16 := new(big.Int).SetBytes(b[16:32])
hash, err := poseidon.Hash([]*big.Int{first16, last16})
if err != nil {
return nil, err
}
return hash, nil
}
func (b *Byte32) Bytes() []byte { return b[:] }
// same action as common.Hash (truncate bytes longer than 32 bytes FROM beginning,
// and padding 0 at the beginning for shorter bytes)
func NewByte32FromBytes(b []byte) *Byte32 {
byte32 := new(Byte32)
if len(b) > 32 {
b = b[len(b)-32:]
}
copy(byte32[32-len(b):], b)
return byte32
}
func NewByte32FromBytesPaddingZero(b []byte) *Byte32 {
if len(b) != 32 && len(b) != 20 {
panic(fmt.Errorf("do not support length except for 120bit and 256bit now. data: %v len: %v", b, len(b)))
}
byte32 := new(Byte32)
copy(byte32[:], b)
return byte32
}

120
core/types/zktrie/hash.go
View file

@ -1,120 +0,0 @@
package zktrie
import (
"encoding/hex"
"fmt"
"math/big"
"github.com/iden3/go-iden3-crypto/utils"
"github.com/scroll-tech/go-ethereum/common"
)
const numCharPrint = 8
// ElemBytesLen is the length of the Hash byte array
const ElemBytesLen = 32
var HashZero = Hash{}
// Hash is the generic type stored in the MerkleTree
type Hash [32]byte
// MarshalText implements the marshaler for the Hash type
func (h Hash) MarshalText() ([]byte, error) {
return []byte(h.BigInt().String()), nil
}
// UnmarshalText implements the unmarshaler for the Hash type
func (h *Hash) UnmarshalText(b []byte) error {
ha, err := NewHashFromString(string(b))
copy(h[:], ha[:])
return err
}
// String returns decimal representation in string format of the Hash
func (h Hash) String() string {
s := h.BigInt().String()
if len(s) < numCharPrint {
return s
}
return s[0:numCharPrint] + "..."
}
// Hex returns the hexadecimal representation of the Hash
func (h Hash) Hex() string {
return hex.EncodeToString(h[:])
}
// BigInt returns the *big.Int representation of the *Hash
func (h *Hash) BigInt() *big.Int {
if new(big.Int).SetBytes(ReverseByteOrder(h[:])) == nil {
return big.NewInt(0)
}
return new(big.Int).SetBytes(ReverseByteOrder(h[:]))
}
// Bytes returns the little endian []byte representation of the *Hash, which always is 32
// bytes length.
func (h *Hash) Bytes() []byte {
b := [32]byte{}
copy(b[:], h[:])
return ReverseByteOrder(b[:])
}
// NewBigIntFromHashBytes returns a *big.Int from a byte array, swapping the
// endianness in the process. This is the intended method to get a *big.Int
// from a byte array that previously has ben generated by the Hash.Bytes()
// method.
func NewBigIntFromHashBytes(b []byte) (*big.Int, error) {
if len(b) != ElemBytesLen {
return nil, fmt.Errorf("expected 32 bytes, found %d bytes", len(b))
}
bi := new(big.Int).SetBytes(b[:ElemBytesLen])
if !utils.CheckBigIntInField(bi) {
return nil, fmt.Errorf("NewBigIntFromHashBytes: Value not inside the Finite Field")
}
return bi, nil
}
// NewHashFromBigInt returns a *Hash representation of the given *big.Int
func NewHashFromBigInt(b *big.Int) *Hash {
r := &Hash{}
copy(r[:], ReverseByteOrder(b.Bytes()))
return r
}
// NewHashFromBytes returns a *Hash from a byte array, swapping the endianness
// in the process. This is the intended method to get a *Hash from a byte array
// that previously has ben generated by the Hash.Bytes() method.
func NewHashFromBytes(b []byte) (*Hash, error) {
if len(b) != ElemBytesLen {
return nil, fmt.Errorf("expected 32 bytes, found %d bytes", len(b))
}
var h Hash
copy(h[:], ReverseByteOrder(b))
return &h, nil
}
// NewHashFromHex returns a *Hash representation of the given hex string
func NewHashFromHex(h string) (*Hash, error) {
return NewHashFromBytes(ReverseByteOrder(common.FromHex(h)))
}
// NewHashFromString returns a *Hash representation of the given decimal string
func NewHashFromString(s string) (*Hash, error) {
bi, ok := new(big.Int).SetString(s, 10)
if !ok {
return nil, fmt.Errorf("can not parse string to Hash")
}
return NewHashFromBigInt(bi), nil
}
// ReverseByteOrder swaps the order of the bytes in the slice.
func ReverseByteOrder(b []byte) []byte {
o := make([]byte, len(b))
for i := range b {
o[len(b)-1-i] = b[i]
}
return o
}

93
core/types/zktrie/util.go
View file

@ -1,93 +0,0 @@
package zktrie
import (
"math/big"
"github.com/scroll-tech/go-ethereum/crypto/poseidon"
)
// HashElems performs a recursive poseidon hash over the array of ElemBytes, each hash
// reduce 2 fieds into one
func HashElems(fst, snd *big.Int, elems ...*big.Int) (*Hash, error) {
l := len(elems)
baseH, err := poseidon.HashFixed([]*big.Int{fst, snd})
if err != nil {
return nil, err
}
if l == 0 {
return NewHashFromBigInt(baseH), nil
} else if l == 1 {
return HashElems(baseH, elems[0])
}
tmp := make([]*big.Int, l/2)
for i := range tmp {
if (i+1)*2 > l {
tmp[i] = elems[i*2+1]
} else {
h, err := poseidon.HashFixed(elems[i*2 : (i+1)*2])
if err != nil {
return nil, err
}
tmp[i] = h
}
}
return HashElems(baseH, tmp[0], tmp[1:]...)
}
// PreHandlingElems turn persisted byte32 elements into field arrays for our hashElem
// it also has the compressed byte32
func PreHandlingElems(flagArray uint32, elems []Byte32) (*Hash, error) {
ret := make([]*big.Int, len(elems))
var err error
for i, elem := range elems {
if flagArray&(1<<i) != 0 {
ret[i], err = elem.Hash()
if err != nil {
return nil, err
}
} else {
ret[i] = new(big.Int).SetBytes(elem[:])
}
}
if len(ret) < 2 {
return NewHashFromBigInt(ret[0]), nil
}
return HashElems(ret[0], ret[1], ret[2:]...)
}
// SetBitBigEndian sets the bit n in the bitmap to 1, in Big Endian.
func SetBitBigEndian(bitmap []byte, n uint) {
bitmap[uint(len(bitmap))-n/8-1] |= 1 << (n % 8)
}
// TestBit tests whether the bit n in bitmap is 1.
func TestBit(bitmap []byte, n uint) bool {
return bitmap[n/8]&(1<<(n%8)) != 0
}
// TestBitBigEndian tests whether the bit n in bitmap is 1, in Big Endian.
func TestBitBigEndian(bitmap []byte, n uint) bool {
return bitmap[uint(len(bitmap))-n/8-1]&(1<<(n%8)) != 0
}
var BigOne = big.NewInt(1)
var BigZero = big.NewInt(0)
func BigEndianBitsToBigInt(bits []bool) *big.Int {
result := big.NewInt(0)
for _, bit := range bits {
result.Mul(result, big.NewInt(2))
if bit {
result.Add(result, BigOne)
}
}
return result
}

1
go.mod
View file

@ -49,6 +49,7 @@ require (
github.com/prometheus/tsdb v0.7.1
github.com/rjeczalik/notify v0.9.1
github.com/rs/cors v1.7.0
github.com/scroll-tech/zktrie v0.2.0
github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible
github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4
github.com/stretchr/testify v1.7.0

2
go.sum
View file

@ -380,6 +380,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/scroll-tech/zktrie v0.2.0 h1:Q0ieI5tKaexihFkNfhSEjiyza2lkQ8LbyKCMMdepKmA=
github.com/scroll-tech/zktrie v0.2.0/go.mod h1:CuJFlG1/soTJJBAySxCZgTF7oPvd5qF6utHOEciC43Q=
github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=

179
trie/zk_trie.go
View file

@ -18,49 +18,50 @@ package trie
import (
"fmt"
"math/big"
zktrie "github.com/scroll-tech/zktrie/trie"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/crypto/poseidon"
"github.com/scroll-tech/go-ethereum/ethdb"
"github.com/scroll-tech/go-ethereum/log"
)
// ZkTrie wraps a trie with key hashing. In a secure trie, all
// access operations hash the key using keccak256. This prevents
// calling code from creating long chains of nodes that
// increase the access time.
//
// Contrary to a regular trie, a ZkTrie can only be created with
// New and must have an attached database. The database also stores
// the preimage of each key.
//
// ZkTrie is not safe for concurrent use.
var magicHash []byte = []byte("THIS IS THE MAGIC INDEX FOR ZKTRIE")
// wrap zktrie for trie interface
type ZkTrie struct {
tree *ZkTrieImpl
*zktrie.ZkTrie
db *ZktrieDatabase
}
func init() {
zkt.InitHashScheme(poseidon.HashFixed)
}
func santiyCheckByte32Key(b []byte) {
if len(b) != 32 && len(b) != 20 {
panic(fmt.Errorf("do not support length except for 120bit and 256bit now. data: %v len: %v", b, len(b)))
}
}
// NewSecure creates a trie
// SecureBinaryTrie bypasses all the buffer mechanism in *Database, it directly uses the
// underlying diskdb
func NewZkTrie(root common.Hash, db *ZktrieDatabase) (*ZkTrie, error) {
rootHash, err := zkt.NewHashFromBytes(root.Bytes())
tr, err := zktrie.NewZkTrie(*zkt.NewByte32FromBytes(root.Bytes()), db)
if err != nil {
return nil, err
}
tree, err := NewZkTrieImplWithRoot((db), rootHash, 256)
if err != nil {
return nil, err
}
return &ZkTrie{
tree: tree,
}, nil
return &ZkTrie{tr, db}, nil
}
// Get returns the value for key stored in the trie.
// The value bytes must not be modified by the caller.
func (t *ZkTrie) Get(key []byte) []byte {
santiyCheckByte32Key(key)
res, err := t.TryGet(key)
if err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
@ -68,45 +69,12 @@ func (t *ZkTrie) Get(key []byte) []byte {
return res
}
// TryGet returns the value for key stored in the trie.
// The value bytes must not be modified by the caller.
// If a node was not found in the database, a MissingNodeError is returned.
func (t *ZkTrie) TryGet(key []byte) ([]byte, error) {
word := zkt.NewByte32FromBytesPaddingZero(key)
k, err := word.Hash()
if err != nil {
return nil, err
}
return t.tree.TryGet(k.Bytes())
}
// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
// possible to use keybyte-encoding as the path might contain odd nibbles.
func (t *ZkTrie) TryGetNode(path []byte) ([]byte, int, error) {
panic("unimplemented")
}
func (t *ZkTrie) updatePreimage(preimage []byte, hashField *big.Int) {
db := t.tree.db.db
if db.preimages != nil { // Ugly direct check but avoids the below write lock
db.lock.Lock()
// we must copy the input key
db.insertPreimage(common.BytesToHash(hashField.Bytes()), common.CopyBytes(preimage))
db.lock.Unlock()
}
}
// TryUpdateAccount will abstract the write of an account to the
// secure trie.
func (t *ZkTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error {
keyPreimage := zkt.NewByte32FromBytesPaddingZero(key)
k, err := keyPreimage.Hash()
if err != nil {
return err
}
t.updatePreimage(key, k)
return t.tree.TryUpdateAccount(k.Bytes(), acc)
santiyCheckByte32Key(key)
value, flag := acc.MarshalFields()
return t.ZkTrie.TryUpdate(key, flag, value)
}
// Update associates key with value in the trie. Subsequent calls to
@ -121,51 +89,21 @@ func (t *ZkTrie) Update(key, value []byte) {
}
}
// TryUpdate associates key with value in the trie. Subsequent calls to
// Get will return value. If value has length zero, any existing value
// is deleted from the trie and calls to Get will return nil.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
//
// If a node was not found in the database, a MissingNodeError is returned.
//
// NOTE: value is restricted to length of bytes32.
// we override the underlying zktrie's TryUpdate method
func (t *ZkTrie) TryUpdate(key, value []byte) error {
keyPreimage := zkt.NewByte32FromBytesPaddingZero(key)
k, err := keyPreimage.Hash()
if err != nil {
return err
}
t.updatePreimage(key, k)
return t.tree.TryUpdate(k.Bytes(), value)
santiyCheckByte32Key(key)
return t.ZkTrie.TryUpdate(key, 1, []zkt.Byte32{*zkt.NewByte32FromBytes(value)})
}
// Delete removes any existing value for key from the trie.
func (t *ZkTrie) Delete(key []byte) {
santiyCheckByte32Key(key)
if err := t.TryDelete(key); err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
}
// TryDelete removes any existing value for key from the trie.
// If a node was not found in the database, a MissingNodeError is returned.
func (t *ZkTrie) TryDelete(key []byte) error {
keyPreimage := zkt.NewByte32FromBytesPaddingZero(key)
k, err := keyPreimage.Hash()
if err != nil {
return err
}
//mitigate the create-delete issue: do not delete unexisted key
if r := t.tree.Get(k.Bytes()); r == nil {
return nil
}
// FIXME: when tryDelete get more solid test, use it instead
return t.tree.tryDeleteLite(zkt.NewHashFromBigInt(k))
}
// GetKey returns the preimage of a hashed key that was
// previously used to store a value.
func (t *ZkTrie) GetKey(kHashBytes []byte) []byte {
@ -175,7 +113,7 @@ func (t *ZkTrie) GetKey(kHashBytes []byte) []byte {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
return t.tree.db.db.preimage(common.BytesToHash(k.Bytes()))
return t.db.db.preimage(common.BytesToHash(k.Bytes()))
}
@ -194,19 +132,13 @@ func (t *ZkTrie) Commit(LeafCallback) (common.Hash, int, error) {
// database and can be used even if the trie doesn't have one.
func (t *ZkTrie) Hash() common.Hash {
var hash common.Hash
hash.SetBytes(t.tree.rootKey.Bytes())
hash.SetBytes(t.ZkTrie.Hash())
return hash
}
// Copy returns a copy of SecureBinaryTrie.
func (t *ZkTrie) Copy() *ZkTrie {
cpy, err := NewZkTrieImplWithRoot(t.tree.db, t.tree.rootKey, t.tree.maxLevels)
if err != nil {
panic("clone trie failed")
}
return &ZkTrie{
tree: cpy,
}
return &ZkTrie{t.ZkTrie.Copy(), t.db}
}
// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration
@ -241,18 +173,13 @@ func (t *ZkTrie) NodeIterator(start []byte) NodeIterator {
// nodes of the longest existing prefix of the key (at least the root node), ending
// with the node that proves the absence of the key.
func (t *ZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error {
word := zkt.NewByte32FromBytesPaddingZero(key)
k, err := word.Hash()
if err != nil {
return err
}
err = t.tree.prove(zkt.NewHashFromBigInt(k), fromLevel, func(n *Node) error {
err := t.ZkTrie.Prove(key, fromLevel, func(n *zktrie.Node) error {
key, err := n.Key()
if err != nil {
return err
}
if n.Type == NodeTypeLeaf {
if n.Type == zktrie.NodeTypeLeaf {
preImage := t.GetKey(n.NodeKey.Bytes())
if len(preImage) > 0 {
n.KeyPreimage = &zkt.Byte32{}
@ -260,7 +187,7 @@ func (t *ZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter)
//return fmt.Errorf("key preimage not found for [%x] ref %x", n.NodeKey.Bytes(), k.Bytes())
}
}
return proofDb.Put(key.Bytes(), n.Value())
return proofDb.Put(key[:], n.Value())
})
if err != nil {
return err
@ -268,5 +195,39 @@ func (t *ZkTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter)
// we put this special kv pair in db so we can distinguish the type and
// make suitable Proof
return proofDb.Put(magicHash, magicSMTBytes)
return proofDb.Put(magicHash, zktrie.ProofMagicBytes())
}
// VerifyProof checks merkle proofs. The given proof must contain the value for
// key in a trie with the given root hash. VerifyProof returns an error if the
// proof contains invalid trie nodes or the wrong value.
func VerifyProofSMT(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) (value []byte, err error) {
h := zkt.NewHashFromBytes(rootHash.Bytes())
k, err := zkt.ToSecureKey(key)
if err != nil {
return nil, err
}
proof, n, err := zktrie.BuildZkTrieProof(h, k, len(key)*8, func(key *zkt.Hash) (*zktrie.Node, error) {
buf, _ := proofDb.Get(key[:])
if buf == nil {
return nil, zktrie.ErrKeyNotFound
}
n, err := zktrie.NewNodeFromBytes(buf)
return n, err
})
if err != nil {
// do not contain the key
return nil, err
} else if !proof.Existence {
return nil, nil
}
if zktrie.VerifyProofZkTrie(h, proof, n) {
return n.Data(), nil
} else {
return nil, fmt.Errorf("bad proof node %v", proof)
}
}

17
trie/zk_trie_database.go
View file

@ -1,14 +1,15 @@
package trie
import (
"math/big"
"github.com/syndtr/goleveldb/leveldb"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/ethdb"
)
// TODO: we should refactor codes, so ZktrieDatabase and Database become two implementation of a
// interface later, making codes less surprising..
// ZktrieDatabase Database adaptor
// ZktrieDatabase Database adaptor imple zktrie.ZktrieDatbase
type ZktrieDatabase struct {
db *Database
prefix []byte
@ -48,6 +49,16 @@ func (l *ZktrieDatabase) Get(key []byte) ([]byte, error) {
return v, err
}
func (l *ZktrieDatabase) UpdatePreimage(preimage []byte, hashField *big.Int) {
db := l.db
if db.preimages != nil { // Ugly direct check but avoids the below write lock
db.lock.Lock()
// we must copy the input key
db.insertPreimage(common.BytesToHash(hashField.Bytes()), common.CopyBytes(preimage))
db.lock.Unlock()
}
}
// Iterate implements the method Iterate of the interface Storage
func (l *ZktrieDatabase) Iterate(f func([]byte, []byte) (bool, error)) error {
iter := l.db.diskdb.NewIterator(l.prefix, nil)

970
trie/zk_trie_impl.go
View file

@ -1,970 +0,0 @@
package trie
import (
"bytes"
"errors"
"fmt"
"io"
cryptoUtils "github.com/iden3/go-iden3-crypto/utils"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/log"
)
const (
// proofFlagsLen is the byte length of the flags in the proof header
// (first 32 bytes).
proofFlagsLen = 2
)
var (
// ErrNodeKeyAlreadyExists is used when a node key already exists.
ErrNodeKeyAlreadyExists = errors.New("key already exists")
// ErrKeyNotFound is used when a key is not found in the ZkTrieImpl.
ErrKeyNotFound = errors.New("key not found in ZkTrieImpl")
// ErrNodeBytesBadSize is used when the data of a node has an incorrect
// size and can't be parsed.
ErrNodeBytesBadSize = errors.New("node data has incorrect size in the DB")
// ErrReachedMaxLevel is used when a traversal of the MT reaches the
// maximum level.
ErrReachedMaxLevel = errors.New("reached maximum level of the merkle tree")
// ErrInvalidNodeFound is used when an invalid node is found and can't
// be parsed.
ErrInvalidNodeFound = errors.New("found an invalid node in the DB")
// ErrInvalidProofBytes is used when a serialized proof is invalid.
ErrInvalidProofBytes = errors.New("the serialized proof is invalid")
// ErrEntryIndexAlreadyExists is used when the entry index already
// exists in the tree.
ErrEntryIndexAlreadyExists = errors.New("the entry index already exists in the tree")
// ErrNotWritable is used when the ZkTrieImpl is not writable and a
// write function is called
ErrNotWritable = errors.New("merkle Tree not writable")
dbKeyRootNode = []byte("currentroot")
)
// ZkTrieImpl is the struct with the main elements of the ZkTrieImpl
type ZkTrieImpl struct {
db *ZktrieDatabase
rootKey *zkt.Hash
writable bool
maxLevels int
Debug bool
}
func NewZkTrieImpl(storage *ZktrieDatabase, maxLevels int) (*ZkTrieImpl, error) {
return NewZkTrieImplWithRoot(storage, &zkt.HashZero, maxLevels)
}
// NewZkTrieImplWithRoot loads a new ZkTrieImpl. If in the storage already exists one
// will open that one, if not, will create a new one.
func NewZkTrieImplWithRoot(storage *ZktrieDatabase, root *zkt.Hash, maxLevels int) (*ZkTrieImpl, error) {
mt := ZkTrieImpl{db: storage, maxLevels: maxLevels, writable: true}
mt.rootKey = root
if *root != zkt.HashZero {
_, err := mt.GetNode(mt.rootKey)
if err != nil {
return nil, err
}
}
return &mt, nil
}
// DB returns the ZkTrieImpl.DB()
func (mt *ZkTrieImpl) DB() *ZktrieDatabase {
return mt.db
}
// Root returns the MerkleRoot
func (mt *ZkTrieImpl) Root() *zkt.Hash {
if mt.Debug {
_, err := mt.GetNode(mt.rootKey)
if err != nil {
var hash common.Hash
hash.SetBytes(mt.rootKey.Bytes())
panic(fmt.Errorf("load trie root failed hash %v", hash))
}
}
return mt.rootKey
}
// MaxLevels returns the MT maximum level
func (mt *ZkTrieImpl) MaxLevels() int {
return mt.maxLevels
}
// tryUpdate update a Key & Value into the ZkTrieImpl. Where the `k` determines the
// path from the Root to the Leaf. This also return the updated leaf node
func (mt *ZkTrieImpl) tryUpdate(kHash *zkt.Hash, vFlag uint32, vPreimage []zkt.Byte32) error {
// verify that the ZkTrieImpl is writable
if !mt.writable {
return ErrNotWritable
}
// verify that k are valid and fit inside the Finite Field.
if !cryptoUtils.CheckBigIntInField(kHash.BigInt()) {
return errors.New("Key not inside the Finite Field")
}
newNodeLeaf := NewNodeLeaf(kHash, vFlag, vPreimage)
path := getPath(mt.maxLevels, kHash[:])
// precalc Key of new leaf here
if _, err := newNodeLeaf.Key(); err != nil {
return err
}
newRootKey, err := mt.addLeaf(newNodeLeaf, mt.rootKey, 0, path, true)
// sanity check
if err == ErrEntryIndexAlreadyExists {
panic("Encounter unexpected errortype: ErrEntryIndexAlreadyExists")
} else if err != nil {
return err
}
mt.rootKey = newRootKey
err = mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
return nil
}
// AddWord
// Deprecated: Add a Bytes32 kv to ZkTrieImpl, only for testing
func (mt *ZkTrieImpl) AddWord(kPreimage, vPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
kHash := zkt.NewHashFromBigInt(k)
newNodeLeaf := NewNodeLeaf(kHash, 1, []zkt.Byte32{*vPreimage})
path := getPath(mt.maxLevels, kHash[:])
// precalc Key of new leaf here
if _, err := newNodeLeaf.Key(); err != nil {
return err
}
newRootKey, err := mt.addLeaf(newNodeLeaf, mt.rootKey, 0, path, false)
if err != nil {
return err
}
mt.rootKey = newRootKey
err = mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
return nil
}
// pushLeaf recursively pushes an existing oldLeaf down until its path diverges
// from newLeaf, at which point both leafs are stored, all while updating the
// path.
func (mt *ZkTrieImpl) pushLeaf(newLeaf *Node, oldLeaf *Node, lvl int,
pathNewLeaf []bool, pathOldLeaf []bool) (*zkt.Hash, error) {
if lvl > mt.maxLevels-2 {
return nil, ErrReachedMaxLevel
}
var newNodeMiddle *Node
if pathNewLeaf[lvl] == pathOldLeaf[lvl] { // We need to go deeper!
nextKey, err := mt.pushLeaf(newLeaf, oldLeaf, lvl+1, pathNewLeaf, pathOldLeaf)
if err != nil {
return nil, err
}
if pathNewLeaf[lvl] { // go right
newNodeMiddle = NewNodeMiddle(&zkt.HashZero, nextKey)
} else { // go left
newNodeMiddle = NewNodeMiddle(nextKey, &zkt.HashZero)
}
return mt.addNode(newNodeMiddle)
}
oldLeafKey, err := oldLeaf.Key()
if err != nil {
return nil, err
}
newLeafKey, err := newLeaf.Key()
if err != nil {
return nil, err
}
if pathNewLeaf[lvl] {
newNodeMiddle = NewNodeMiddle(oldLeafKey, newLeafKey)
} else {
newNodeMiddle = NewNodeMiddle(newLeafKey, oldLeafKey)
}
// We can add newLeaf now. We don't need to add oldLeaf because it's
// already in the tree.
_, err = mt.addNode(newLeaf)
if err != nil {
return nil, err
}
return mt.addNode(newNodeMiddle)
}
// addLeaf recursively adds a newLeaf in the MT while updating the path.
func (mt *ZkTrieImpl) addLeaf(newLeaf *Node, key *zkt.Hash,
lvl int, path []bool, forceUpdate bool) (*zkt.Hash, error) {
var err error
var nextKey *zkt.Hash
if lvl > mt.maxLevels-1 {
return nil, ErrReachedMaxLevel
}
n, err := mt.GetNode(key)
if err != nil {
fmt.Printf("addLeaf:GetNode err %v key %v root %v level %v\n", err, key, mt.rootKey, lvl)
fmt.Printf("root %v\n", mt.Root())
return nil, err
}
switch n.Type {
case NodeTypeEmpty:
// We can add newLeaf now
{
r, e := mt.addNode(newLeaf)
if e != nil {
fmt.Println("err on NodeTypeEmpty mt.addNode ", e)
}
return r, e
}
case NodeTypeLeaf:
// Check if leaf node found contains the leaf node we are
// trying to add
if bytes.Equal(n.NodeKey[:], newLeaf.NodeKey[:]) {
k, err := n.Key()
if err != nil {
fmt.Println("err on obtain key of duplicated entry", err)
return nil, err
}
if bytes.Equal(k[:], newLeaf.key[:]) {
// do nothing, duplicate entry
// FIXME more optimization may needed here
return k, nil
} else if forceUpdate {
return mt.updateNode(newLeaf)
}
fmt.Printf("ErrEntryIndexAlreadyExists nodeKey %v n.Key() %v newLeaf.Key() %v\n",
n.NodeKey, k, newLeaf.key)
return nil, ErrEntryIndexAlreadyExists
}
pathOldLeaf := getPath(mt.maxLevels, n.NodeKey[:])
// We need to push newLeaf down until its path diverges from
// n's path
return mt.pushLeaf(newLeaf, n, lvl, path, pathOldLeaf)
case NodeTypeMiddle:
// We need to go deeper, continue traversing the tree, left or
// right depending on path
var newNodeMiddle *Node
if path[lvl] { // go right
nextKey, err = mt.addLeaf(newLeaf, n.ChildR, lvl+1, path, forceUpdate)
newNodeMiddle = NewNodeMiddle(n.ChildL, nextKey)
} else { // go left
nextKey, err = mt.addLeaf(newLeaf, n.ChildL, lvl+1, path, forceUpdate)
newNodeMiddle = NewNodeMiddle(nextKey, n.ChildR)
}
if err != nil {
fmt.Printf("addLeaf:GetNode err %v level %v\n", err, lvl)
return nil, err
}
// Update the node to reflect the modified child
return mt.addNode(newNodeMiddle)
default:
return nil, ErrInvalidNodeFound
}
}
// addNode adds a node into the MT. Empty nodes are not stored in the tree;
// they are all the same and assumed to always exist.
func (mt *ZkTrieImpl) addNode(n *Node) (*zkt.Hash, error) {
// verify that the ZkTrieImpl is writable
if !mt.writable {
return nil, ErrNotWritable
}
if n.Type == NodeTypeEmpty {
return n.Key()
}
k, err := n.Key()
if err != nil {
return nil, err
}
v := n.Value()
// Check that the node key doesn't already exist
oldV, err := mt.db.Get(k[:])
if err == nil {
if !bytes.Equal(oldV, v) {
return nil, ErrNodeKeyAlreadyExists
} else {
// duplicated
return k, nil
}
}
err = mt.db.Put(k[:], v)
return k, err
}
// updateNode updates an existing node in the MT. Empty nodes are not stored
// in the tree; they are all the same and assumed to always exist.
func (mt *ZkTrieImpl) updateNode(n *Node) (*zkt.Hash, error) {
// verify that the ZkTrieImpl is writable
if !mt.writable {
return nil, ErrNotWritable
}
if n.Type == NodeTypeEmpty {
return n.Key()
}
k, err := n.Key()
if err != nil {
return nil, err
}
v := n.Value()
err = mt.db.Put(k[:], v)
return k, err
}
/*
// Get returns the value of the leaf for the given key
func (mt *ZkTrieImpl) Get(k *big.Int) (*big.Int, *big.Int, []*zkt.Hash, error) {
// verify that k is valid and fit inside the Finite Field.
if !cryptoUtils.CheckBigIntInField(k) {
return nil, nil, nil, errors.New("Key not inside the Finite Field")
}
kHash := zkt.NewHashFromBigInt(k)
path := getPath(mt.maxLevels, kHash[:])
nextKey := mt.rootKey
siblings := []*zkt.Hash{}
for i := 0; i < mt.maxLevels; i++ {
n, err := mt.GetNode(nextKey)
if err != nil {
return nil, nil, nil, err
}
switch n.Type {
case NodeTypeEmpty:
return big.NewInt(0), big.NewInt(0), siblings, ErrKeyNotFound
case NodeTypeLeaf:
if bytes.Equal(kHash[:], n.NodeKey[:]) {
return n.Entry[0].BigInt(), n.Entry[1].BigInt(), siblings, nil
}
return n.Entry[0].BigInt(), n.Entry[1].BigInt(), siblings, ErrKeyNotFound
case NodeTypeMiddle:
if path[i] {
nextKey = n.ChildR
siblings = append(siblings, n.ChildL)
} else {
nextKey = n.ChildL
siblings = append(siblings, n.ChildR)
}
default:
return nil, nil, nil, ErrInvalidNodeFound
}
}
return nil, nil, nil, ErrReachedMaxLevel
}
*/
func (mt *ZkTrieImpl) tryGet(kHash *zkt.Hash) (*Node, []*zkt.Hash, error) {
path := getPath(mt.maxLevels, kHash[:])
nextKey := mt.rootKey
var siblings []*zkt.Hash
for i := 0; i < mt.maxLevels; i++ {
n, err := mt.GetNode(nextKey)
if err != nil {
return nil, nil, err
}
switch n.Type {
case NodeTypeEmpty:
return NewNodeEmpty(), siblings, ErrKeyNotFound
case NodeTypeLeaf:
if bytes.Equal(kHash[:], n.NodeKey[:]) {
return n, siblings, nil
}
return n, siblings, ErrKeyNotFound
case NodeTypeMiddle:
if path[i] {
nextKey = n.ChildR
siblings = append(siblings, n.ChildL)
} else {
nextKey = n.ChildL
siblings = append(siblings, n.ChildR)
}
default:
return nil, nil, ErrInvalidNodeFound
}
}
return nil, siblings, ErrReachedMaxLevel
}
// Get returns the value for key stored in the trie.
// The value bytes must not be modified by the caller.
func (mt *ZkTrieImpl) Get(key []byte) []byte {
res, err := mt.TryGet(key)
if err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
return res
}
// TryGet returns the value for key stored in the trie.
// The value bytes must not be modified by the caller.
// If a node was not found in the database, a MissingNodeError is returned.
func (mt *ZkTrieImpl) TryGet(key []byte) ([]byte, error) {
kHash, err := zkt.NewHashFromBytes(common.BytesToHash(key).Bytes())
if err != nil {
return nil, err
}
node, _, err := mt.tryGet(kHash)
if err == ErrKeyNotFound {
// according to https://github.com/ethereum/go-ethereum/blob/37f9d25ba027356457953eab5f181c98b46e9988/trie/trie.go#L135
return nil, nil
} else if err != nil {
return nil, err
}
return node.Data(), nil
}
// GetLeafNodeByWord
// Deprecated: Get a Bytes32 kv to ZkTrieImpl, only for testing
func (mt *ZkTrieImpl) GetLeafNodeByWord(kPreimage *zkt.Byte32) (*Node, error) {
k, err := kPreimage.Hash()
if err != nil {
return nil, err
}
n, _, err := mt.tryGet(zkt.NewHashFromBigInt(k))
return n, err
}
// Deprecated: only for testing
func (mt *ZkTrieImpl) UpdateWord(kPreimage, vPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
err = mt.tryUpdate(zkt.NewHashFromBigInt(k), 1, []zkt.Byte32{*vPreimage})
return err
}
// TryUpdateAccount will abstract the write of an account to the trie
func (mt *ZkTrieImpl) TryUpdateAccount(key []byte, acc *types.StateAccount) error {
kHash, err := zkt.NewHashFromBytes(common.BytesToHash(key).Bytes())
if err != nil {
return err
}
value, flag := acc.MarshalFields()
return mt.tryUpdate(kHash, flag, value)
}
// Update associates key with value in the trie. Subsequent calls to
// Get will return value.
// TODO: If value has length zero, any existing value
// is deleted from the trie and calls to Get will return nil.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
func (mt *ZkTrieImpl) Update(key, value []byte) {
if err := mt.TryUpdate(key, value); err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
}
// TryUpdate associates key with value in the trie. Subsequent calls to
// Get will return value.
// TODO: If value has length zero, any existing value
// is deleted from the trie and calls to Get will return nil.
//
// The value bytes must not be modified by the caller while they are
// stored in the trie.
//
// Currently the value must shorter than 32 bytes
//
// NOTE: value is restricted to length of bytes32.
func (mt *ZkTrieImpl) TryUpdate(key, value []byte) error {
kHash, err := zkt.NewHashFromBytes(common.BytesToHash(key).Bytes())
if err != nil {
return err
}
vPreimage := zkt.NewByte32FromBytesPaddingZero(value)
return mt.tryUpdate(kHash, 1, []zkt.Byte32{*vPreimage})
}
/*
func (mt *ZkTrieImpl) UpdateVarWord(kPreimage *zkt.Byte32, vHash *big.Int, vPreimage []byte) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
err = mt.Update(k, vHash, kPreimage, vPreimage[:])
if err == ErrKeyNotFound {
err = mt.Add(k, vHash, kPreimage, vPreimage[:])
if err != nil {
log.Error("UpdateVarWord, inset still failed %v root %v", err, mt.rootKey)
}
return err
} else if err != nil {
log.Error("UpdateVarWord err %v %v", err, reflect.TypeOf(err))
}
return err
}
*/
// only update the corresponding leaf for
func (mt *ZkTrieImpl) tryDeleteLite(kHash *zkt.Hash) error {
// verify that the ZkTrieImpl is writable
if !mt.writable {
return ErrNotWritable
}
// verify that k is valid and fit inside the Finite Field.
if !cryptoUtils.CheckBigIntInField(kHash.BigInt()) {
return errors.New("Key not inside the Finite Field")
}
path := getPath(mt.maxLevels, kHash[:])
nextKey := mt.rootKey
siblings := []*zkt.Hash{}
for i := 0; i < mt.maxLevels; i++ {
n, err := mt.GetNode(nextKey)
if err != nil {
return err
}
switch n.Type {
case NodeTypeEmpty:
return ErrKeyNotFound
case NodeTypeLeaf:
if bytes.Equal(kHash[:], n.NodeKey[:]) {
// remove and go up with the sibling
newRootKey, err := mt.recalculatePathUntilRoot(path, NewNodeEmpty(), siblings)
if err != nil {
return err
}
mt.rootKey = newRootKey
return mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
}
return ErrKeyNotFound
case NodeTypeMiddle:
if path[i] {
nextKey = n.ChildR
siblings = append(siblings, n.ChildL)
} else {
nextKey = n.ChildL
siblings = append(siblings, n.ChildR)
}
default:
return ErrInvalidNodeFound
}
}
return ErrKeyNotFound
}
// Delete removes the specified Key from the ZkTrieImpl and updates the path
// from the deleted key to the Root with the new values. This method removes
// the key from the ZkTrieImpl, but does not remove the old nodes from the
// key-value database; this means that if the tree is accessed by an old Root
// where the key was not deleted yet, the key will still exist. If is desired
// to remove the key-values from the database that are not under the current
// Root, an option could be to dump all the leafs (using mt.DumpLeafs) and
// import them in a new ZkTrieImpl in a new database (using
// mt.ImportDumpedLeafs), but this will lose all the Root history of the
// ZkTrieImpl
func (mt *ZkTrieImpl) tryDelete(kHash *zkt.Hash) error {
// verify that the ZkTrieImpl is writable
if !mt.writable {
return ErrNotWritable
}
// verify that k is valid and fit inside the Finite Field.
if !cryptoUtils.CheckBigIntInField(kHash.BigInt()) {
return errors.New("Key not inside the Finite Field")
}
path := getPath(mt.maxLevels, kHash[:])
nextKey := mt.rootKey
siblings := []*zkt.Hash{}
for i := 0; i < mt.maxLevels; i++ {
n, err := mt.GetNode(nextKey)
if err != nil {
return err
}
switch n.Type {
case NodeTypeEmpty:
return ErrKeyNotFound
case NodeTypeLeaf:
if bytes.Equal(kHash[:], n.NodeKey[:]) {
// remove and go up with the sibling
err = mt.rmAndUpload(path, kHash, siblings)
return err
}
return ErrKeyNotFound
case NodeTypeMiddle:
if path[i] {
nextKey = n.ChildR
siblings = append(siblings, n.ChildL)
} else {
nextKey = n.ChildL
siblings = append(siblings, n.ChildR)
}
default:
return ErrInvalidNodeFound
}
}
return ErrKeyNotFound
}
// Delete removes the specified Key from the ZkTrieImpl and updates the path
// from the deleted key to the Root with the new values. This method removes
// the key from the ZkTrieImpl, but does not remove the old nodes from the
// key-value database; this means that if the tree is accessed by an old Root
// where the key was not deleted yet, the key will still exist. If is desired
// to remove the key-values from the database that are not under the current
// Root, an option could be to dump all the leafs (using mt.DumpLeafs) and
// import them in a new ZkTrieImpl in a new database (using
// mt.ImportDumpedLeafs), but this will lose all the Root history of the
// ZkTrieImpl
// Delete removes any existing value for key from the trie.
func (t *ZkTrieImpl) Delete(key []byte) {
if err := t.TryDelete(key); err != nil {
log.Error(fmt.Sprintf("Unhandled trie error: %v", err))
}
}
// TryDelete removes any existing value for key from the trie.
// If a node was not found in the database, a MissingNodeError is returned.
func (mt *ZkTrieImpl) TryDelete(key []byte) error {
kHash, err := zkt.NewHashFromBytes(common.BytesToHash(key).Bytes())
if err != nil {
return err
}
return mt.tryDelete(kHash)
}
// Deprecated: only for testing
func (mt *ZkTrieImpl) DeleteWord(kPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
return mt.tryDelete(zkt.NewHashFromBigInt(k))
}
// rmAndUpload removes the key, and goes up until the root updating all the
// nodes with the new values.
func (mt *ZkTrieImpl) rmAndUpload(path []bool, kHash *zkt.Hash, siblings []*zkt.Hash) error {
if len(siblings) == 0 {
mt.rootKey = &zkt.HashZero
err := mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
return nil
}
toUpload := siblings[len(siblings)-1]
if len(siblings) < 2 { //nolint:gomnd
mt.rootKey = siblings[0]
err := mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
return nil
}
for i := len(siblings) - 2; i >= 0; i-- { //nolint:gomnd
if !bytes.Equal(siblings[i][:], zkt.HashZero[:]) {
var newNode *Node
if path[i] {
newNode = NewNodeMiddle(siblings[i], toUpload)
} else {
newNode = NewNodeMiddle(toUpload, siblings[i])
}
_, err := mt.addNode(newNode)
if err != ErrNodeKeyAlreadyExists && err != nil {
return err
}
// go up until the root
newRootKey, err := mt.recalculatePathUntilRoot(path, newNode,
siblings[:i])
if err != nil {
return err
}
mt.rootKey = newRootKey
err = mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
break
}
// if i==0 (root position), stop and store the sibling of the
// deleted leaf as root
if i == 0 {
mt.rootKey = toUpload
err := mt.dbInsert(dbKeyRootNode, DBEntryTypeRoot, mt.rootKey[:])
if err != nil {
return err
}
break
}
}
return nil
}
// recalculatePathUntilRoot recalculates the nodes until the Root
func (mt *ZkTrieImpl) recalculatePathUntilRoot(path []bool, node *Node,
siblings []*zkt.Hash) (*zkt.Hash, error) {
for i := len(siblings) - 1; i >= 0; i-- {
nodeKey, err := node.Key()
if err != nil {
return nil, err
}
if path[i] {
node = NewNodeMiddle(siblings[i], nodeKey)
} else {
node = NewNodeMiddle(nodeKey, siblings[i])
}
_, err = mt.addNode(node)
if err != ErrNodeKeyAlreadyExists && err != nil {
return nil, err
}
}
// return last node added, which is the root
nodeKey, err := node.Key()
return nodeKey, err
}
// dbInsert is a helper function to insert a node into a key in an open db
// transaction.
func (mt *ZkTrieImpl) dbInsert(k []byte, t NodeType, data []byte) error {
v := append([]byte{byte(t)}, data...)
return mt.db.Put(k, v)
}
// GetNode gets a node by key from the MT. Empty nodes are not stored in the
// tree; they are all the same and assumed to always exist.
// <del>for non exist key, return (NewNodeEmpty(), nil)</del>
func (mt *ZkTrieImpl) GetNode(key *zkt.Hash) (*Node, error) {
if bytes.Equal(key[:], zkt.HashZero[:]) {
return NewNodeEmpty(), nil
}
nBytes, err := mt.db.Get(key[:])
if err == ErrNotFound {
//return NewNodeEmpty(), nil
return nil, ErrKeyNotFound
} else if err != nil {
return nil, err
}
return NewNodeFromBytes(nBytes)
}
// getPath returns the binary path, from the root to the leaf.
func getPath(numLevels int, k []byte) []bool {
path := make([]bool, numLevels)
for n := 0; n < numLevels; n++ {
path[n] = zkt.TestBit(k[:], uint(n))
}
return path
}
// NodeAux contains the auxiliary node used in a non-existence proof.
type NodeAux struct {
Key *zkt.Hash
Value *zkt.Hash
}
// Proof defines the required elements for a MT proof of existence or
// non-existence.
type Proof struct {
// existence indicates wether this is a proof of existence or
// non-existence.
Existence bool
// depth indicates how deep in the tree the proof goes.
depth uint
// notempties is a bitmap of non-empty Siblings found in Siblings.
notempties [zkt.ElemBytesLen - proofFlagsLen]byte
// Siblings is a list of non-empty sibling keys.
Siblings []*zkt.Hash
// Key is the key of leaf in existence case
Key *zkt.Hash
NodeAux *NodeAux
}
// VerifyProof verifies the Merkle Proof for the entry and root.
func VerifyProofZkTrie(rootKey *zkt.Hash, proof *Proof, node *Node) bool {
key, err := node.Key()
if err != nil {
return false
}
rootFromProof, err := proof.Verify(key, node.NodeKey)
if err != nil {
return false
}
return bytes.Equal(rootKey[:], rootFromProof[:])
}
// Verify the proof and calculate the root, key can be nil when try to verify
// an nonexistent proof
func (proof *Proof) Verify(key, kHash *zkt.Hash) (*zkt.Hash, error) {
if proof.Existence {
if key == nil {
return nil, ErrKeyNotFound
}
return proof.rootFromProof(key, kHash)
} else {
if proof.NodeAux == nil {
return proof.rootFromProof(&zkt.HashZero, kHash)
} else {
if bytes.Equal(kHash[:], proof.NodeAux.Key[:]) {
return nil, fmt.Errorf("non-existence proof being checked against hIndex equal to nodeAux")
}
midKey, err := LeafKey(proof.NodeAux.Key, proof.NodeAux.Value)
if err != nil {
return nil, err
}
return proof.rootFromProof(midKey, kHash)
}
}
}
func (proof *Proof) rootFromProof(key, kHash *zkt.Hash) (*zkt.Hash, error) {
midKey := key
var err error
sibIdx := len(proof.Siblings) - 1
path := getPath(int(proof.depth), kHash[:])
var siblingKey *zkt.Hash
for lvl := int(proof.depth) - 1; lvl >= 0; lvl-- {
if zkt.TestBitBigEndian(proof.notempties[:], uint(lvl)) {
siblingKey = proof.Siblings[sibIdx]
sibIdx--
} else {
siblingKey = &zkt.HashZero
}
if path[lvl] {
midKey, err = NewNodeMiddle(siblingKey, midKey).Key()
if err != nil {
return nil, err
}
} else {
midKey, err = NewNodeMiddle(midKey, siblingKey).Key()
if err != nil {
return nil, err
}
}
}
return midKey, nil
}
// walk is a helper recursive function to iterate over all tree branches
func (mt *ZkTrieImpl) walk(key *zkt.Hash, f func(*Node)) error {
n, err := mt.GetNode(key)
if err != nil {
return err
}
switch n.Type {
case NodeTypeEmpty:
f(n)
case NodeTypeLeaf:
f(n)
case NodeTypeMiddle:
f(n)
if err := mt.walk(n.ChildL, f); err != nil {
return err
}
if err := mt.walk(n.ChildR, f); err != nil {
return err
}
default:
return ErrInvalidNodeFound
}
return nil
}
// Walk iterates over all the branches of a ZkTrieImpl with the given rootKey
// if rootKey is nil, it will get the current RootKey of the current state of
// the ZkTrieImpl. For each node, it calls the f function given in the
// parameters. See some examples of the Walk function usage in the
// ZkTrieImpl.go and merkletree_test.go
func (mt *ZkTrieImpl) Walk(rootKey *zkt.Hash, f func(*Node)) error {
if rootKey == nil {
rootKey = mt.Root()
}
err := mt.walk(rootKey, f)
return err
}
// GraphViz uses Walk function to generate a string GraphViz representation of
// the tree and writes it to w
func (mt *ZkTrieImpl) GraphViz(w io.Writer, rootKey *zkt.Hash) error {
fmt.Fprintf(w, `digraph hierarchy {
node [fontname=Monospace,fontsize=10,shape=box]
`)
cnt := 0
var errIn error
err := mt.Walk(rootKey, func(n *Node) {
k, err := n.Key()
if err != nil {
errIn = err
}
switch n.Type {
case NodeTypeEmpty:
case NodeTypeLeaf:
fmt.Fprintf(w, "\"%v\" [style=filled];\n", k.String())
case NodeTypeMiddle:
lr := [2]string{n.ChildL.String(), n.ChildR.String()}
emptyNodes := ""
for i := range lr {
if lr[i] == "0" {
lr[i] = fmt.Sprintf("empty%v", cnt)
emptyNodes += fmt.Sprintf("\"%v\" [style=dashed,label=0];\n", lr[i])
cnt++
}
}
fmt.Fprintf(w, "\"%v\" -> {\"%v\" \"%v\"}\n", k.String(), lr[0], lr[1])
fmt.Fprint(w, emptyNodes)
default:
}
})
fmt.Fprintf(w, "}\n")
if errIn != nil {
return errIn
}
return err
}
// PrintGraphViz prints directly the GraphViz() output
func (mt *ZkTrieImpl) PrintGraphViz(rootKey *zkt.Hash) error {
if rootKey == nil {
rootKey = mt.Root()
}
w := bytes.NewBufferString("")
fmt.Fprintf(w,
"--------\nGraphViz of the ZkTrieImpl with RootKey "+rootKey.BigInt().String()+"\n")
err := mt.GraphViz(w, nil)
if err != nil {
return err
}
fmt.Fprintf(w,
"End of GraphViz of the ZkTrieImpl with RootKey "+rootKey.BigInt().String()+"\n--------\n")
fmt.Println(w)
return nil
}

104
trie/zk_trie_impl_test.go
View file

@ -9,18 +9,100 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
zktrie "github.com/scroll-tech/zktrie/trie"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/core/types"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/ethdb/memorydb"
)
// we do not need zktrie impl anymore, only made a wrapper for adapting testing
type zkTrieImplTestWrapper struct {
*zktrie.ZkTrieImpl
}
func newZkTrieImpl(storage *ZktrieDatabase, maxLevels int) (*zkTrieImplTestWrapper, error) {
return newZkTrieImplWithRoot(storage, &zkt.HashZero, maxLevels)
}
// NewZkTrieImplWithRoot loads a new ZkTrieImpl. If in the storage already exists one
// will open that one, if not, will create a new one.
func newZkTrieImplWithRoot(storage *ZktrieDatabase, root *zkt.Hash, maxLevels int) (*zkTrieImplTestWrapper, error) {
impl, err := zktrie.NewZkTrieImplWithRoot(storage, root, maxLevels)
if err != nil {
return nil, err
}
return &zkTrieImplTestWrapper{impl}, nil
}
// AddWord
// Deprecated: Add a Bytes32 kv to ZkTrieImpl, only for testing
func (mt *zkTrieImplTestWrapper) AddWord(kPreimage, vPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
if v, _ := mt.TryGet(k.Bytes()); v != nil {
return zktrie.ErrEntryIndexAlreadyExists
}
return mt.ZkTrieImpl.TryUpdate(zkt.NewHashFromBigInt(k), 1, []zkt.Byte32{*vPreimage})
}
// GetLeafNodeByWord
// Deprecated: Get a Bytes32 kv to ZkTrieImpl, only for testing
func (mt *zkTrieImplTestWrapper) GetLeafNodeByWord(kPreimage *zkt.Byte32) (*zktrie.Node, error) {
k, err := kPreimage.Hash()
if err != nil {
return nil, err
}
return mt.ZkTrieImpl.GetLeafNode(zkt.NewHashFromBigInt(k))
}
// Deprecated: only for testing
func (mt *zkTrieImplTestWrapper) UpdateWord(kPreimage, vPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
return mt.ZkTrieImpl.TryUpdate(zkt.NewHashFromBigInt(k), 1, []zkt.Byte32{*vPreimage})
}
// Deprecated: only for testing
func (mt *zkTrieImplTestWrapper) DeleteWord(kPreimage *zkt.Byte32) error {
k, err := kPreimage.Hash()
if err != nil {
return err
}
return mt.ZkTrieImpl.TryDelete(zkt.NewHashFromBigInt(k))
}
func (mt *zkTrieImplTestWrapper) TryGet(key []byte) ([]byte, error) {
return mt.ZkTrieImpl.TryGet(zkt.NewHashFromBytes(key))
}
// TryUpdateAccount will abstract the write of an account to the trie
func (mt *zkTrieImplTestWrapper) TryUpdateAccount(key []byte, acc *types.StateAccount) error {
value, flag := acc.MarshalFields()
return mt.ZkTrieImpl.TryUpdate(zkt.NewHashFromBytes(key), flag, value)
}
// NewHashFromHex returns a *Hash representation of the given hex string
func NewHashFromHex(h string) (*zkt.Hash, error) {
return zkt.NewHashFromCheckedBytes(common.FromHex(h))
}
type Fatalable interface {
Fatal(args ...interface{})
}
func newTestingMerkle(f Fatalable, numLevels int) *ZkTrieImpl {
mt, err := NewZkTrieImpl(NewZktrieDatabase((memorydb.New())), numLevels)
func newTestingMerkle(f Fatalable, numLevels int) *zkTrieImplTestWrapper {
mt, err := newZkTrieImpl(NewZktrieDatabase((memorydb.New())), numLevels)
if err != nil {
f.Fatal(err)
return nil
@ -46,20 +128,20 @@ func TestHashParsers(t *testing.T) {
h := zkt.NewHashFromBigInt(b)
assert.Equal(t, "4932297968297298434239270129193057052722409868268166443802652458940273154854", h.BigInt().String()) //nolint:lll
assert.Equal(t, "49322979...", h.String())
assert.Equal(t, "265baaf161e875c372d08e50f52abddc01d32efc93e90290bb8b3d9ceb94e70a", h.Hex())
assert.Equal(t, "0ae794eb9c3d8bbb9002e993fc2ed301dcbd2af5508ed072c375e861f1aa5b26", h.Hex())
b1, err := zkt.NewBigIntFromHashBytes(b.Bytes())
assert.Nil(t, err)
assert.Equal(t, new(big.Int).SetBytes(b.Bytes()).String(), b1.String())
b2, err := zkt.NewHashFromBytes(b.Bytes())
b2, err := zkt.NewHashFromCheckedBytes(b.Bytes())
assert.Nil(t, err)
assert.Equal(t, b.String(), b2.BigInt().String())
h2, err := zkt.NewHashFromHex(h.Hex())
h2, err := NewHashFromHex(h.Hex())
assert.Nil(t, err)
assert.Equal(t, h, h2)
_, err = zkt.NewHashFromHex("0x12")
_, err = NewHashFromHex("0x12")
assert.NotNil(t, err)
// check limits
@ -73,12 +155,12 @@ func testHashParsers(t *testing.T, a *big.Int) {
require.True(t, cryptoUtils.CheckBigIntInField(a))
h := zkt.NewHashFromBigInt(a)
assert.Equal(t, a, h.BigInt())
hFromBytes, err := zkt.NewHashFromBytes(h.Bytes())
hFromBytes, err := zkt.NewHashFromCheckedBytes(h.Bytes())
assert.Nil(t, err)
assert.Equal(t, h, hFromBytes)
assert.Equal(t, a, hFromBytes.BigInt())
assert.Equal(t, a.String(), hFromBytes.BigInt().String())
hFromHex, err := zkt.NewHashFromHex(h.Hex())
hFromHex, err := NewHashFromHex(h.Hex())
assert.Nil(t, err)
assert.Equal(t, h, hFromHex)
@ -97,7 +179,7 @@ func TestMerkleTree_AddUpdateGetWord(t *testing.T) {
err = mt.AddWord(&zkt.Byte32{5}, &zkt.Byte32{6})
assert.Nil(t, err)
err = mt.AddWord(&zkt.Byte32{5}, &zkt.Byte32{7})
assert.Equal(t, ErrEntryIndexAlreadyExists, err)
assert.Equal(t, zktrie.ErrEntryIndexAlreadyExists, err)
node, err := mt.GetLeafNodeByWord(&zkt.Byte32{1})
assert.Nil(t, err)
@ -132,7 +214,7 @@ func TestMerkleTree_AddUpdateGetWord(t *testing.T) {
assert.Equal(t, len(node.ValuePreimage), 1)
assert.Equal(t, (&zkt.Byte32{9})[:], node.ValuePreimage[0][:])
_, err = mt.GetLeafNodeByWord(&zkt.Byte32{100})
assert.Equal(t, ErrKeyNotFound, err)
assert.Equal(t, zktrie.ErrKeyNotFound, err)
}
func TestMerkleTree_UpdateAccount(t *testing.T) {

222
trie/zk_trie_node.go
View file

@ -1,222 +0,0 @@
package trie
import (
"encoding/binary"
"fmt"
"math/big"
"reflect"
"unsafe"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
)
// NodeType defines the type of node in the MT.
type NodeType byte
const (
// NodeTypeMiddle indicates the type of middle Node that has children.
NodeTypeMiddle NodeType = 0
// NodeTypeLeaf indicates the type of a leaf Node that contains a key &
// value.
NodeTypeLeaf NodeType = 1
// NodeTypeEmpty indicates the type of an empty Node.
NodeTypeEmpty NodeType = 2
// DBEntryTypeRoot indicates the type of a DB entry that indicates the
// current Root of a MerkleTree
DBEntryTypeRoot NodeType = 3
)
// Node is the struct that represents a node in the MT. The node should not be
// modified after creation because the cached key won't be updated.
type Node struct {
// Type is the type of node in the tree.
Type NodeType
// ChildL is the left child of a middle node.
ChildL *zkt.Hash
// ChildR is the right child of a middle node.
ChildR *zkt.Hash
// NodeKey is the node's key stored in a leaf node.
NodeKey *zkt.Hash
// ValuePreimage can store at most 256 byte32 as fields (represnted by BIG-ENDIAN integer)
// and the first 24 can be compressed (each bytes32 consider as 2 fields), in hashing the compressed
// elemments would be calculated first
ValuePreimage []zkt.Byte32
// CompressedFlags use each bit for indicating the compressed flag for the first 24 fields
CompressedFlags uint32
// key is the cache of entry key used to avoid recalculating
key *zkt.Hash
// valueHash is the cache of hashes of valuePreimage, used to avoid recalculating
valueHash *zkt.Hash
// KeyPreimage is kept here only for proof
KeyPreimage *zkt.Byte32
}
// NewNodeLeaf creates a new leaf node.
func NewNodeLeaf(k *zkt.Hash, valueFlags uint32, valuePreimage []zkt.Byte32) *Node {
return &Node{Type: NodeTypeLeaf, NodeKey: k, CompressedFlags: valueFlags, ValuePreimage: valuePreimage}
}
// NewNodeMiddle creates a new middle node.
func NewNodeMiddle(childL *zkt.Hash, childR *zkt.Hash) *Node {
return &Node{Type: NodeTypeMiddle, ChildL: childL, ChildR: childR}
}
// NewNodeEmpty creates a new empty node.
func NewNodeEmpty() *Node {
return &Node{Type: NodeTypeEmpty}
}
// NewNodeFromBytes creates a new node by parsing the input []byte.
func NewNodeFromBytes(b []byte) (*Node, error) {
if len(b) < 1 {
return nil, ErrNodeBytesBadSize
}
n := Node{Type: NodeType(b[0])}
b = b[1:]
switch n.Type {
case NodeTypeMiddle:
if len(b) != 2*zkt.ElemBytesLen {
return nil, ErrNodeBytesBadSize
}
n.ChildL, _ = zkt.NewHashFromBytes(b[:zkt.ElemBytesLen])
n.ChildR, _ = zkt.NewHashFromBytes(b[zkt.ElemBytesLen : zkt.ElemBytesLen*2])
case NodeTypeLeaf:
if len(b) < zkt.ElemBytesLen+4 {
return nil, ErrNodeBytesBadSize
}
n.NodeKey, _ = zkt.NewHashFromBytes(b[0:32])
mark := binary.LittleEndian.Uint32(b[32:36])
preimageLen := int(mark & 255)
n.CompressedFlags = mark >> 8
n.ValuePreimage = make([]zkt.Byte32, preimageLen)
curPos := 36
for i := 0; i < preimageLen; i++ {
copy(n.ValuePreimage[i][:], b[i*32+curPos:(i+1)*32+curPos])
}
curPos = 36 + preimageLen*32
preImageSize := int(b[curPos])
curPos += 1
if preImageSize != 0 {
n.KeyPreimage = new(zkt.Byte32)
copy(n.KeyPreimage[:], b[curPos:curPos+preImageSize])
}
case NodeTypeEmpty:
break
default:
return nil, ErrInvalidNodeFound
}
return &n, nil
}
// LeafKey computes the key of a leaf node given the hIndex and hValue of the
// entry of the leaf.
func LeafKey(k, v *zkt.Hash) (*zkt.Hash, error) {
return zkt.HashElems(big.NewInt(1), k.BigInt(), v.BigInt())
}
// Key computes the key of the node by hashing the content in a specific way
// for each type of node. This key is used as the hash of the merklee tree for
// each node.
func (n *Node) Key() (*zkt.Hash, error) {
if n.key == nil { // Cache the key to avoid repeated hash computations.
// NOTE: We are not using the type to calculate the hash!
switch n.Type {
case NodeTypeMiddle: // H(ChildL || ChildR)
var err error
n.key, err = zkt.HashElems(n.ChildL.BigInt(), n.ChildR.BigInt())
if err != nil {
return nil, err
}
case NodeTypeLeaf:
var err error
n.valueHash, err = zkt.PreHandlingElems(n.CompressedFlags, n.ValuePreimage)
if err != nil {
return nil, err
}
n.key, err = LeafKey(n.NodeKey, n.valueHash)
if err != nil {
return nil, err
}
case NodeTypeEmpty: // Zero
n.key = &zkt.HashZero
default:
n.key = &zkt.HashZero
}
}
return n.key, nil
}
func (n *Node) ValueKey() (*zkt.Hash, error) {
if _, err := n.Key(); err != nil {
return nil, err
}
return n.valueHash, nil
}
// Data returns the wrapped data inside LeafNode and cast them into bytes
// for other node type it just return nil
func (n *Node) Data() []byte {
switch n.Type {
case NodeTypeLeaf:
var data []byte
hdata := (*reflect.SliceHeader)(unsafe.Pointer(&data))
//TODO: uintptr(reflect.ValueOf(n.ValuePreimage).UnsafePointer()) should be more elegant but only available until go 1.18
hdata.Data = uintptr(unsafe.Pointer(&n.ValuePreimage[0]))
hdata.Len = 32 * len(n.ValuePreimage)
hdata.Cap = hdata.Len
return data
default:
return nil
}
}
// Value returns the value of the node. This is the content that is stored in
// the backend database.
func (n *Node) Value() []byte {
switch n.Type {
case NodeTypeMiddle: // {Type || ChildL || ChildR}
bytes := []byte{byte(n.Type)}
bytes = append(bytes, n.ChildL.Bytes()...)
bytes = append(bytes, n.ChildR.Bytes()...)
return bytes
case NodeTypeLeaf: // {Type || Data...}
bytes := []byte{byte(n.Type)}
bytes = append(bytes, n.NodeKey.Bytes()...)
tmp := make([]byte, 4)
compressedFlag := (n.CompressedFlags << 8) + uint32(len(n.ValuePreimage))
binary.LittleEndian.PutUint32(tmp, compressedFlag)
bytes = append(bytes, tmp...)
for _, elm := range n.ValuePreimage {
bytes = append(bytes, elm[:]...)
}
if n.KeyPreimage != nil {
bytes = append(bytes, byte(len(n.KeyPreimage)))
bytes = append(bytes, n.KeyPreimage[:]...)
} else {
bytes = append(bytes, 0)
}
return bytes
case NodeTypeEmpty: // { Type }
return []byte{byte(n.Type)}
default:
return []byte{}
}
}
// String outputs a string representation of a node (different for each type).
func (n *Node) String() string {
switch n.Type {
case NodeTypeMiddle: // {Type || ChildL || ChildR}
return fmt.Sprintf("Middle L:%s R:%s", n.ChildL, n.ChildR)
case NodeTypeLeaf: // {Type || Data...}
return fmt.Sprintf("Leaf I:%v Items: %d, First:%v", n.NodeKey, len(n.ValuePreimage), n.ValuePreimage[0])
case NodeTypeEmpty: // {}
return "Empty"
default:
return "Invalid Node"
}
}

195
trie/zk_trie_proof.go
View file

@ -1,195 +0,0 @@
package trie
import (
"bytes"
"fmt"
"math/big"
"github.com/scroll-tech/go-ethereum/common"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/ethdb"
)
// TODO: remove this hack
var magicHash []byte
var magicSMTBytes []byte
func init() {
magicSMTBytes = []byte("THIS IS SOME MAGIC BYTES FOR SMT m1rRXgP2xpDI")
hasher := newHasher(false)
defer returnHasherToPool(hasher)
magicHash = hasher.hashData(magicSMTBytes)
}
// Prove constructs a merkle proof for SMT, it respect the protocol used by the ethereum-trie
// but save the node data with a compact form
func (mt *ZkTrieImpl) prove(kHash *zkt.Hash, fromLevel uint, writeNode func(*Node) error) error {
path := getPath(mt.maxLevels, kHash[:])
var nodes []*Node
tn := mt.rootKey
for i := 0; i < mt.maxLevels; i++ {
n, err := mt.GetNode(tn)
if err != nil {
return err
}
finished := true
switch n.Type {
case NodeTypeEmpty:
case NodeTypeLeaf:
// notice even we found a leaf whose entry didn't match the expected k,
// we still include it as the proof of absence
case NodeTypeMiddle:
finished = false
if path[i] {
tn = n.ChildR
} else {
tn = n.ChildL
}
default:
return ErrInvalidNodeFound
}
nodes = append(nodes, n)
if finished {
break
}
}
for _, n := range nodes {
if fromLevel > 0 {
fromLevel--
continue
}
// TODO: notice here we may have broken some implicit on the proofDb:
// the key is not kecca(value) and it even can not be derived from
// the value by any means without a actually decoding
if err := writeNode(n); err != nil {
return err
}
}
return nil
}
func (mt *ZkTrieImpl) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error {
kHash, err := zkt.NewHashFromBytes(common.BytesToHash(key).Bytes())
if err != nil {
return err
}
err = mt.prove(kHash, fromLevel, func(n *Node) error {
key, err := n.Key()
if err != nil {
return err
}
return proofDb.Put(key[:], n.Value())
})
if err != nil {
return err
}
// we put this special kv pair in db so we can distinguish the type and
// make suitable Proof
return proofDb.Put(magicHash, magicSMTBytes)
}
func buildZkTrieProof(rootKey *zkt.Hash, k *big.Int, lvl int, getNode func(key *zkt.Hash) (*Node, error)) (*Proof,
*Node, error) {
p := &Proof{}
var siblingKey *zkt.Hash
kHash := zkt.NewHashFromBigInt(k)
path := getPath(lvl, kHash[:])
nextKey := rootKey
for p.depth = 0; p.depth < uint(lvl); p.depth++ {
n, err := getNode(nextKey)
if err != nil {
return nil, nil, err
}
switch n.Type {
case NodeTypeEmpty:
return p, n, nil
case NodeTypeLeaf:
if bytes.Equal(kHash[:], n.NodeKey[:]) {
p.Existence = true
return p, n, nil
}
// We found a leaf whose entry didn't match hIndex
p.NodeAux = &NodeAux{Key: n.NodeKey, Value: n.valueHash}
return p, n, nil
case NodeTypeMiddle:
if path[p.depth] {
nextKey = n.ChildR
siblingKey = n.ChildL
} else {
nextKey = n.ChildL
siblingKey = n.ChildR
}
default:
return nil, nil, ErrInvalidNodeFound
}
if !bytes.Equal(siblingKey[:], zkt.HashZero[:]) {
zkt.SetBitBigEndian(p.notempties[:], p.depth)
p.Siblings = append(p.Siblings, siblingKey)
}
}
return nil, nil, ErrKeyNotFound
}
// DecodeProof try to decode a node bytes, return can be nil for any non-node data (magic code)
func DecodeSMTProof(data []byte) (*Node, error) {
if bytes.Equal(magicSMTBytes, data) {
//skip magic bytes node
return nil, nil
}
return NewNodeFromBytes(data)
}
// VerifyProof checks merkle proofs. The given proof must contain the value for
// key in a trie with the given root hash. VerifyProof returns an error if the
// proof contains invalid trie nodes or the wrong value.
func VerifyProofSMT(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) (value []byte, err error) {
h, err := zkt.NewHashFromBytes(rootHash.Bytes())
if err != nil {
return nil, err
}
word := zkt.NewByte32FromBytesPaddingZero(key)
k, err := word.Hash()
if err != nil {
return nil, err
}
proof, n, err := buildZkTrieProof(h, k, len(key)*8, func(key *zkt.Hash) (*Node, error) {
buf, _ := proofDb.Get(key[:])
if buf == nil {
return nil, ErrKeyNotFound
}
n, err := NewNodeFromBytes(buf)
return n, err
})
if err != nil {
// do not contain the key
return nil, err
} else if !proof.Existence {
return nil, nil
}
if VerifyProofZkTrie(h, proof, n) {
return n.Data(), nil
} else {
return nil, fmt.Errorf("bad proof node %v", proof)
}
}

32
trie/zk_trie_proof_test.go
View file

@ -24,8 +24,9 @@ import (
"github.com/stretchr/testify/assert"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/crypto"
"github.com/scroll-tech/go-ethereum/ethdb/memorydb"
)
@ -36,7 +37,7 @@ func init() {
// makeProvers creates Merkle trie provers based on different implementations to
// test all variations.
func makeSMTProvers(mt *ZkTrieImpl) []func(key []byte) *memorydb.Database {
func makeSMTProvers(mt *ZkTrie) []func(key []byte) *memorydb.Database {
var provers []func(key []byte) *memorydb.Database
// Create a direct trie based Merkle prover
@ -47,7 +48,11 @@ func makeSMTProvers(mt *ZkTrieImpl) []func(key []byte) *memorydb.Database {
panic(err)
}
proof := memorydb.New()
mt.Prove(k.Bytes(), 0, proof)
err = mt.Prove(common.BytesToHash(k.Bytes()).Bytes(), 0, proof)
if err != nil {
panic(err)
}
return proof
})
return provers
@ -58,13 +63,14 @@ func verifyValue(proveVal []byte, vPreimage []byte) bool {
}
func TestSMTOneElementProof(t *testing.T) {
mt, _ := NewZkTrieImpl(NewZktrieDatabase((memorydb.New())), 64)
tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New())))
mt := &zkTrieImplTestWrapper{tr.Tree()}
err := mt.UpdateWord(
zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("k"), 32)),
zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("v"), 32)),
)
assert.Nil(t, err)
for i, prover := range makeSMTProvers(mt) {
for i, prover := range makeSMTProvers(tr) {
keyBytes := bytes.Repeat([]byte("k"), 32)
proof := prover(keyBytes)
if proof == nil {
@ -85,7 +91,7 @@ func TestSMTOneElementProof(t *testing.T) {
func TestSMTProof(t *testing.T) {
mt, vals := randomZktrie(t, 500)
root := mt.Root()
root := mt.Tree().Root()
for i, prover := range makeSMTProvers(mt) {
for _, kv := range vals {
proof := prover(kv.k)
@ -105,7 +111,7 @@ func TestSMTProof(t *testing.T) {
func TestSMTBadProof(t *testing.T) {
mt, vals := randomZktrie(t, 500)
root := mt.Root()
root := mt.Tree().Root()
for i, prover := range makeSMTProvers(mt) {
for _, kv := range vals {
proof := prover(kv.k)
@ -134,14 +140,15 @@ func TestSMTBadProof(t *testing.T) {
// Tests that missing keys can also be proven. The test explicitly uses a single
// entry trie and checks for missing keys both before and after the single entry.
func TestSMTMissingKeyProof(t *testing.T) {
mt, _ := NewZkTrieImpl(NewZktrieDatabase((memorydb.New())), 64)
tr, _ := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New())))
mt := &zkTrieImplTestWrapper{tr.Tree()}
err := mt.UpdateWord(
zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("k"), 20)),
zkt.NewByte32FromBytesPaddingZero(bytes.Repeat([]byte("v"), 20)),
)
assert.Nil(t, err)
prover := makeSMTProvers(mt)[0]
prover := makeSMTProvers(tr)[0]
for i, key := range []string{"a", "j", "l", "z"} {
keyBytes := bytes.Repeat([]byte(key), 32)
@ -160,11 +167,12 @@ func TestSMTMissingKeyProof(t *testing.T) {
}
}
func randomZktrie(t *testing.T, n int) (*ZkTrieImpl, map[string]*kv) {
mt, err := NewZkTrieImpl(NewZktrieDatabase((memorydb.New())), 64)
func randomZktrie(t *testing.T, n int) (*ZkTrie, map[string]*kv) {
tr, err := NewZkTrie(common.Hash{}, NewZktrieDatabase((memorydb.New())))
if err != nil {
panic(err)
}
mt := &zkTrieImplTestWrapper{tr.Tree()}
vals := make(map[string]*kv)
for i := byte(0); i < 100; i++ {
@ -185,5 +193,5 @@ func randomZktrie(t *testing.T, n int) (*ZkTrieImpl, map[string]*kv) {
vals[string(value.k)] = value
}
return mt, vals
return tr, vals
}

3
trie/zk_trie_test.go
View file

@ -24,8 +24,9 @@ import (
"github.com/stretchr/testify/assert"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/ethdb/memorydb"
)

37
trie/zkproof/writer.go
View file

@ -6,10 +6,12 @@ import (
"fmt"
"math/big"
zktrie "github.com/scroll-tech/zktrie/trie"
zkt "github.com/scroll-tech/zktrie/types"
"github.com/scroll-tech/go-ethereum/common"
"github.com/scroll-tech/go-ethereum/common/hexutil"
"github.com/scroll-tech/go-ethereum/core/types"
zkt "github.com/scroll-tech/go-ethereum/core/types/zktrie"
"github.com/scroll-tech/go-ethereum/ethdb/memorydb"
"github.com/scroll-tech/go-ethereum/log"
"github.com/scroll-tech/go-ethereum/trie"
@ -39,10 +41,10 @@ func addressToKey(addr common.Address) *zkt.Hash {
}
//resume the proof bytes into db and return the leaf node
func resumeProofs(proof []hexutil.Bytes, db *memorydb.Database) *trie.Node {
func resumeProofs(proof []hexutil.Bytes, db *memorydb.Database) *zktrie.Node {
for _, buf := range proof {
n, err := trie.DecodeSMTProof(buf)
n, err := zktrie.DecodeSMTProof(buf)
if err != nil {
log.Warn("decode proof string fail", "error", err)
} else if n != nil {
@ -53,7 +55,7 @@ func resumeProofs(proof []hexutil.Bytes, db *memorydb.Database) *trie.Node {
//notice: must consistent with trie/merkletree.go
bt := k[:]
db.Put(bt, buf)
if n.Type == trie.NodeTypeLeaf || n.Type == trie.NodeTypeEmpty {
if n.Type == zktrie.NodeTypeLeaf || n.Type == zktrie.NodeTypeEmpty {
return n
}
}
@ -68,14 +70,14 @@ func resumeProofs(proof []hexutil.Bytes, db *memorydb.Database) *trie.Node {
// whole path in sequence, from root to leaf
func decodeProofForMPTPath(proof proofList, path *SMTPath) {
var lastNode *trie.Node
var lastNode *zktrie.Node
keyPath := big.NewInt(0)
path.KeyPathPart = (*hexutil.Big)(keyPath)
keyCounter := big.NewInt(1)
for _, buf := range proof {
n, err := trie.DecodeSMTProof(buf)
n, err := zktrie.DecodeSMTProof(buf)
if err != nil {
log.Warn("decode proof string fail", "error", err)
} else if n != nil {
@ -105,9 +107,9 @@ func decodeProofForMPTPath(proof proofList, path *SMTPath) {
keyCounter.Mul(keyCounter, big.NewInt(2))
}
switch n.Type {
case trie.NodeTypeMiddle:
case zktrie.NodeTypeMiddle:
lastNode = n
case trie.NodeTypeLeaf:
case zktrie.NodeTypeLeaf:
vhash, _ := n.ValueKey()
path.Leaf = &SMTPathNode{
//here we just return the inner represent of hash (little endian, reversed byte order to common hash)
@ -125,7 +127,7 @@ func decodeProofForMPTPath(proof proofList, path *SMTPath) {
}
return
case trie.NodeTypeEmpty:
case zktrie.NodeTypeEmpty:
return
default:
panic(fmt.Errorf("unknown node type %d", n.Type))
@ -158,7 +160,7 @@ func NewZkTrieProofWriter(storage *types.StorageTrace) (*zktrieProofWriter, erro
for addrs, proof := range storage.Proofs {
if n := resumeProofs(proof, underlayerDb); n != nil {
addr := common.HexToAddress(addrs)
if n.Type == trie.NodeTypeEmpty {
if n.Type == zktrie.NodeTypeEmpty {
accounts[addr] = nil
} else if acc, err := types.UnmarshalStateAccount(n.Data()); err == nil {
if bytes.Equal(n.NodeKey[:], addressToKey(addr)[:]) {
@ -371,7 +373,8 @@ func (w *zktrieProofWriter) traceAccountUpdate(addr common.Address, updateAccDat
}
var proof proofList
if err := w.tracingZktrie.Prove(addr.Bytes32(), 0, &proof); err != nil {
s_key, _ := zkt.ToSecureKeyBytes(addr.Bytes())
if err := w.tracingZktrie.Prove(s_key.Bytes(), 0, &proof); err != nil {
return nil, fmt.Errorf("prove BEFORE state for <%x> fail: %s", addr.Bytes(), err)
}
@ -412,7 +415,7 @@ func (w *zktrieProofWriter) traceAccountUpdate(addr common.Address, updateAccDat
}
proof = proofList{}
if err := w.tracingZktrie.Prove(addr.Bytes32(), 0, &proof); err != nil {
if err := w.tracingZktrie.Prove(s_key.Bytes(), 0, &proof); err != nil {
return nil, fmt.Errorf("prove AFTER state fail: %s", err)
}
@ -465,7 +468,8 @@ func (w *zktrieProofWriter) traceStorageUpdate(addr common.Address, key, value [
}
var storageBeforeProof, storageAfterProof proofList
if err := trie.Prove(storeKey.Bytes(), 0, &storageBeforeProof); err != nil {
s_key, _ := zkt.ToSecureKeyBytes(storeKey.Bytes())
if err := trie.Prove(s_key.Bytes(), 0, &storageBeforeProof); err != nil {
return nil, fmt.Errorf("prove BEFORE storage state fail: %s", err)
}
@ -488,7 +492,7 @@ func (w *zktrieProofWriter) traceStorageUpdate(addr common.Address, key, value [
}
}
if err := trie.Prove(storeKey.Bytes(), 0, &storageAfterProof); err != nil {
if err := trie.Prove(s_key.Bytes(), 0, &storageAfterProof); err != nil {
return nil, fmt.Errorf("prove AFTER storage state fail: %s", err)
}
decodeProofForMPTPath(storageAfterProof, statePath[1])
@ -561,10 +565,7 @@ func (w *zktrieProofWriter) HandleNewState(accountState *types.AccountWrapper) (
return nil, fmt.Errorf("update account state %s fail: %s", accountState.Address, err)
}
hash, err := zkt.NewHashFromBytes(stateRoot[:])
if err != nil {
return nil, fmt.Errorf("malform of state root in account %s", accountState.Address)
}
hash := zkt.NewHashFromBytes(stateRoot[:])
out.CommonStateRoot = hash[:]
return out, nil
}