wip: special storage key/val encoding

build/checksums.txt

@@ -2,8 +2,8 @@
 
 # version:spec-tests v5.1.0
 # https://github.com/ethereum/execution-spec-tests/releases
-# https://github.com/ethereum/execution-spec-tests/releases/download/bal%40v1.8.0
+# https://github.com/ethereum/execution-spec-tests/releases/download/bal%40v2.0.0
-af50623f0555715e61fbadff15bd621229591c13be010ca3ea6e4b1b217c926c fixtures_bal.tar.gz
+68b2f29a3e3a794b5a9a23692190847f177e0a7a24891ccc7d7ad267ca59e4e0 fixtures_bal.tar.gz
 
 # version:golang 1.25.1
 # https://go.dev/dl/

core/block_access_list_tracer.go

@@ -95,7 +95,7 @@ func (a *BlockAccessListTracer) OnNonceChange(addr common.Address, prev uint64,
 }
 
 func (a *BlockAccessListTracer) OnStorageRead(addr common.Address, key common.Hash) {
-	a.builder.StorageRead(addr, key)
+	a.builder.StorageRead(addr, bal.Storage(key))
 }
 
 func (a *BlockAccessListTracer) OnAcountRead(addr common.Address) {
@@ -103,5 +103,5 @@ func (a *BlockAccessListTracer) OnAcountRead(addr common.Address) {
 }
 
 func (a *BlockAccessListTracer) OnStorageChange(addr common.Address, slot common.Hash, prev common.Hash, new common.Hash) {
-	a.builder.StorageWrite(addr, slot, prev, new)
+	a.builder.StorageWrite(addr, bal.Storage(slot), bal.Storage(prev), bal.Storage(new))
 }

core/state/bal_reader.go

@@ -106,7 +106,7 @@ func (r *BALReader) initObjFromDiff(db *StateDB, addr common.Address, a *types.S
 	}
 	if diff.StorageWrites != nil {
 		for key, val := range diff.StorageWrites {
-			obj.pendingStorage[key] = val
+			obj.pendingStorage[common.Hash(key)] = common.Hash(val)
 		}
 	}
 	if obj.empty() {
@@ -227,7 +227,7 @@ func (r *BALReader) accountChangesAt(addr common.Address, idx int) *bal.AccountM
 
 	for i := len(acct.StorageChanges) - 1; i >= 0; i-- {
 		if res.StorageWrites == nil {
-			res.StorageWrites = make(map[common.Hash]common.Hash)
+			res.StorageWrites = make(map[bal.Storage]bal.Storage)
 		}
 		slotWrites := acct.StorageChanges[i]
 
@@ -288,7 +288,7 @@ func (r *BALReader) readAccountDiff(addr common.Address, idx int) *bal.AccountMu
 	}
 
 	if len(diff.StorageChanges) > 0 {
-		res.StorageWrites = make(map[common.Hash]common.Hash)
+		res.StorageWrites = make(map[bal.Storage]bal.Storage)
 		for _, slotWrites := range diff.StorageChanges {
 			for i := 0; i < len(slotWrites.Accesses) && slotWrites.Accesses[i].TxIdx <= uint16(idx); i++ {
 				res.StorageWrites[slotWrites.Slot] = slotWrites.Accesses[i].ValueAfter

core/state/bal_state_transition.go

@@ -197,11 +197,11 @@ func (s *BALStateTransition) commitAccount(addr common.Address) (*accountUpdate,
 	op.storagesOriginByKey = make(map[common.Hash][]byte)
 
 	for key, value := range s.diffs[addr].StorageWrites {
-		hash := crypto.Keccak256Hash(key.Bytes())
+		hash := crypto.Keccak256Hash(key[:])
-		op.storages[hash] = encode(value)
+		op.storages[hash] = encode(common.Hash(value))
-		origin := encode(s.originStorages[addr][key])
+		origin := encode(s.originStorages[addr][common.Hash(key)])
 		op.storagesOriginByHash[hash] = origin
-		op.storagesOriginByKey[key] = origin
+		op.storagesOriginByKey[common.Hash(key)] = origin
 	}
 	tr, _ := s.tries.Load(addr)
 	root, nodes := tr.(Trie).Commit(false)
@@ -377,14 +377,14 @@ func (s *BALStateTransition) loadOriginStorages() {
 		for key := range diff.StorageWrites {
 			storageKey := key
 			go func() {
-				val, err := s.reader.Storage(addr, storageKey)
+				val, err := s.reader.Storage(addr, common.Hash(storageKey))
 				if err != nil {
 					s.setError(err)
 					return
 				}
 				originStoragesCh <- &originStorage{
 					addr,
-					storageKey,
+					common.Hash(storageKey),
 					val,
 				}
 			}()
@@ -469,7 +469,7 @@ func (s *BALStateTransition) IntermediateRoot(_ bool) common.Hash {
 					deleteKeys               [][]byte
 				)
 				for key, val := range diff.StorageWrites {
-					if val != (common.Hash{}) {
+					if val != (bal.Storage{}) {
 						updateKeys = append(updateKeys, key[:])
 						updateValues = append(updateValues, common.TrimLeftZeroes(val[:]))
 

core/types/bal/bal.go

@@ -55,7 +55,7 @@ func newAccessListBuilder(logger *slog.Logger) *idxAccessListBuilder {
 	}
 }
 
-func (c *idxAccessListBuilder) storageRead(address common.Address, key common.Hash) {
+func (c *idxAccessListBuilder) storageRead(address common.Address, key Storage) {
 	if _, ok := c.accessesStack[len(c.accessesStack)-1][address]; !ok {
 		c.accessesStack[len(c.accessesStack)-1][address] = &constructionAccountAccess{}
 	}
@@ -69,12 +69,12 @@ func (c *idxAccessListBuilder) accountRead(address common.Address) {
 	}
 }
 
-func (c *idxAccessListBuilder) storageWrite(address common.Address, key, prevVal, newVal common.Hash) {
+func (c *idxAccessListBuilder) storageWrite(address common.Address, key, prevVal, newVal Storage) {
 	if _, ok := c.prestates[address]; !ok {
 		c.prestates[address] = &accountIdxPrestate{}
 	}
 	if c.prestates[address].storage == nil {
-		c.prestates[address].storage = make(map[common.Hash]common.Hash)
+		c.prestates[address].storage = make(map[Storage]Storage)
 	}
 	if _, ok := c.prestates[address].storage[key]; !ok {
 		c.prestates[address].storage[key] = prevVal
@@ -134,7 +134,7 @@ func (c *idxAccessListBuilder) codeChange(address common.Address, prev, cur []by
 func (c *idxAccessListBuilder) selfDestruct(address common.Address) {
 	access := c.accessesStack[len(c.accessesStack)-1][address]
 	if len(access.storageMutations) != 0 && access.storageReads == nil {
-		access.storageReads = make(map[common.Hash]struct{})
+		access.storageReads = make(map[Storage]struct{})
 	}
 	for key, _ := range access.storageMutations {
 		access.storageReads[key] = struct{}{}
@@ -219,7 +219,7 @@ func (a *idxAccessListBuilder) finalise() (*StateDiff, StateAccesses) {
 		// if the account has no net mutations against the index prestate, only include
 		// it in the state read set
 		if len(access.code) == 0 && access.nonce == nil && access.balance == nil && len(access.storageMutations) == 0 {
-			stateAccesses[addr] = make(map[common.Hash]struct{})
+			stateAccesses[addr] = make(map[Storage]struct{})
 			if access.storageReads != nil {
 				stateAccesses[addr] = access.storageReads
 			}
@@ -276,11 +276,11 @@ func (c *AccessListBuilder) FinaliseIdxChanges(idx uint16) {
 		}
 		if pendingAcctDiff.StorageWrites != nil {
 			if finalizedAcctChanges.StorageWrites == nil {
-				finalizedAcctChanges.StorageWrites = make(map[common.Hash]map[uint16]common.Hash)
+				finalizedAcctChanges.StorageWrites = make(map[Storage]map[uint16]Storage)
 			}
 			for key, val := range pendingAcctDiff.StorageWrites {
 				if _, ok := finalizedAcctChanges.StorageWrites[key]; !ok {
-					finalizedAcctChanges.StorageWrites[key] = make(map[uint16]common.Hash)
+					finalizedAcctChanges.StorageWrites[key] = make(map[uint16]Storage)
 				}
 				finalizedAcctChanges.StorageWrites[key][idx] = val
 
@@ -310,7 +310,7 @@ func (c *AccessListBuilder) FinaliseIdxChanges(idx uint16) {
 				continue
 			}
 			if finalizedAcctAccesses.StorageReads == nil {
-				finalizedAcctAccesses.StorageReads = make(map[common.Hash]struct{})
+				finalizedAcctAccesses.StorageReads = make(map[Storage]struct{})
 			}
 			finalizedAcctAccesses.StorageReads[key] = struct{}{}
 		}
@@ -319,13 +319,13 @@ func (c *AccessListBuilder) FinaliseIdxChanges(idx uint16) {
 	c.lastFinalizedAccesses = pendingAccesses
 }
 
-func (c *AccessListBuilder) StorageRead(address common.Address, key common.Hash) {
+func (c *AccessListBuilder) StorageRead(address common.Address, key Storage) {
 	c.idxBuilder.storageRead(address, key)
 }
 func (c *AccessListBuilder) AccountRead(address common.Address) {
 	c.idxBuilder.accountRead(address)
 }
-func (c *AccessListBuilder) StorageWrite(address common.Address, key, prevVal, newVal common.Hash) {
+func (c *AccessListBuilder) StorageWrite(address common.Address, key, prevVal, newVal Storage) {
 	c.idxBuilder.storageWrite(address, key, prevVal, newVal)
 }
 func (c *AccessListBuilder) BalanceChange(address common.Address, prev, cur *uint256.Int) {
@@ -362,14 +362,14 @@ type ConstructionAccountAccesses struct {
 	// StorageWrites is the post-state values of an account's storage slots
 	// that were modified in a block, keyed by the slot key and the tx index
 	// where the modification occurred.
-	StorageWrites map[common.Hash]map[uint16]common.Hash
+	StorageWrites map[Storage]map[uint16]Storage
 
 	// StorageReads is the set of slot keys that were accessed during block
 	// execution.
 	//
 	// Storage slots which are both read and written (with changed values)
 	// appear only in StorageWrites.
-	StorageReads map[common.Hash]struct{}
+	StorageReads map[Storage]struct{}
 
 	// BalanceChanges contains the post-transaction balances of an account,
 	// keyed by transaction indices where it was changed.
@@ -391,8 +391,8 @@ type constructionAccountAccess struct {
 	nonce   *uint64
 	balance *uint256.Int
 
-	storageMutations map[common.Hash]common.Hash
+	storageMutations map[Storage]Storage
-	storageReads     map[common.Hash]struct{}
+	storageReads     map[Storage]struct{}
 }
 
 // Merge adds the accesses/mutations from other into the calling instance. If
@@ -408,7 +408,7 @@ func (c *constructionAccountAccess) Merge(other *constructionAccountAccess) {
 	}
 	if other.storageMutations != nil {
 		if c.storageMutations == nil {
-			c.storageMutations = make(map[common.Hash]common.Hash)
+			c.storageMutations = make(map[Storage]Storage)
 		}
 		for key, val := range other.storageMutations {
 			c.storageMutations[key] = val
@@ -417,7 +417,7 @@ func (c *constructionAccountAccess) Merge(other *constructionAccountAccess) {
 	}
 	if other.storageReads != nil {
 		if c.storageReads == nil {
-			c.storageReads = make(map[common.Hash]struct{})
+			c.storageReads = make(map[Storage]struct{})
 		}
 		// TODO: if the state was mutated in the caller, don't add it to the caller's reads.
 		// need to have a test case for this, verify it fails in the current state, and then fix this bug.
@@ -433,7 +433,7 @@ func (c *constructionAccountAccess) Merge(other *constructionAccountAccess) {
 func (c *constructionAccountAccess) MergeReads(other *constructionAccountAccess) {
 	if other.storageMutations != nil {
 		if c.storageReads == nil {
-			c.storageReads = make(map[common.Hash]struct{})
+			c.storageReads = make(map[Storage]struct{})
 		}
 		for key, _ := range other.storageMutations {
 			if _, ok := c.storageMutations[key]; ok {
@@ -444,7 +444,7 @@ func (c *constructionAccountAccess) MergeReads(other *constructionAccountAccess)
 	}
 	if other.storageReads != nil {
 		if c.storageReads == nil {
-			c.storageReads = make(map[common.Hash]struct{})
+			c.storageReads = make(map[Storage]struct{})
 		}
 		for key := range other.storageReads {
 			if _, ok := c.storageMutations[key]; ok {
@@ -455,18 +455,18 @@ func (c *constructionAccountAccess) MergeReads(other *constructionAccountAccess)
 	}
 }
 
-func (c *constructionAccountAccess) StorageRead(key common.Hash) {
+func (c *constructionAccountAccess) StorageRead(key Storage) {
 	if c.storageReads == nil {
-		c.storageReads = make(map[common.Hash]struct{})
+		c.storageReads = make(map[Storage]struct{})
 	}
 	if _, ok := c.storageMutations[key]; !ok {
 		c.storageReads[key] = struct{}{}
 	}
 }
 
-func (c *constructionAccountAccess) StorageWrite(key, prevVal, newVal common.Hash) {
+func (c *constructionAccountAccess) StorageWrite(key, prevVal, newVal Storage) {
 	if c.storageMutations == nil {
-		c.storageMutations = make(map[common.Hash]common.Hash)
+		c.storageMutations = make(map[Storage]Storage)
 	}
 	c.storageMutations[key] = newVal
 	// a key can be first read and later written, but it must only show up
@@ -518,7 +518,7 @@ func (c *AccessListBuilder) Copy() *AccessListBuilder {
 	for addr, aa := range c.FinalizedAccesses {
 		var aaCopy ConstructionAccountAccesses
 
-		slotWrites := make(map[common.Hash]map[uint16]common.Hash, len(aa.StorageWrites))
+		slotWrites := make(map[Storage]map[uint16]Storage, len(aa.StorageWrites))
 		for key, m := range aa.StorageWrites {
 			slotWrites[key] = maps.Clone(m)
 		}
@@ -558,13 +558,13 @@ type StateDiff struct {
 
 // StateAccesses contains a set of accounts/storage that were accessed during the
 // execution of one or more access list indices.
-type StateAccesses map[common.Address]map[common.Hash]struct{}
+type StateAccesses map[common.Address]map[Storage]struct{}
 
 // Merge combines adds the accesses from other into s.
 func (s *StateAccesses) Merge(other StateAccesses) {
 	for addr, accesses := range other {
 		if _, ok := (*s)[addr]; !ok {
-			(*s)[addr] = make(map[common.Hash]struct{})
+			(*s)[addr] = make(map[Storage]struct{})
 		}
 		for slot := range accesses {
 			(*s)[addr][slot] = struct{}{}
@@ -578,16 +578,16 @@ type accountIdxPrestate struct {
 	balance *uint256.Int
 	nonce   *uint64
 	code    ContractCode
-	storage map[common.Hash]common.Hash
+	storage map[Storage]Storage
 }
 
 // AccountMutations contains mutations that were made to an account across
 // one or more access list indices.
 type AccountMutations struct {
-	Balance       *uint256.Int                `json:"Balance,omitempty"`
+	Balance       *uint256.Int        `json:"Balance,omitempty"`
-	Nonce         *uint64                     `json:"Nonce,omitempty"`
+	Nonce         *uint64             `json:"Nonce,omitempty"`
-	Code          ContractCode                `json:"Code,omitempty"`
+	Code          ContractCode        `json:"Code,omitempty"`
-	StorageWrites map[common.Hash]common.Hash `json:"StorageWrites,omitempty"`
+	StorageWrites map[Storage]Storage `json:"StorageWrites,omitempty"`
 }
 
 // String returns a human-readable JSON representation of the account mutations.
@@ -618,7 +618,7 @@ func (a *AccountMutations) LogDiff(addr common.Address, other *AccountMutations)
 
 	if a.StorageWrites != nil || other.StorageWrites != nil {
 		if !maps.Equal(a.StorageWrites, other.StorageWrites) {
-			union := make(map[common.Hash]struct{})
+			union := make(map[Storage]struct{})
 			for slot, _ := range a.StorageWrites {
 				union[slot] = struct{}{}
 			}
@@ -626,7 +626,7 @@ func (a *AccountMutations) LogDiff(addr common.Address, other *AccountMutations)
 				union[slot] = struct{}{}
 			}
 
-			orderedKeys := slices.SortedFunc(maps.Keys(union), func(hash common.Hash, hash2 common.Hash) int {
+			orderedKeys := slices.SortedFunc(maps.Keys(union), func(hash Storage, hash2 Storage) int {
 				return bytes.Compare(hash[:], hash2[:])
 			})
 

core/types/bal/bal_encoding.go

@@ -19,12 +19,14 @@ package bal
 import (
 	"bytes"
 	"cmp"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"maps"
 	"slices"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -140,13 +142,67 @@ type encodingAccountNonce struct {
 
 // encodingStorageWrite is the encoding format of StorageWrites.
 type encodingStorageWrite struct {
-	TxIdx      uint16      `json:"txIndex"`
+	TxIdx      uint16  `json:"txIndex"`
-	ValueAfter common.Hash `json:"valueAfter"`
+	ValueAfter Storage `json:"valueAfter"`
+}
+
+// Storage can represent either a storage key or value
+type Storage common.Hash
+
+// Cmp compares two hashes.
+func (h Storage) Cmp(other Storage) int {
+	return bytes.Compare(h[:], other[:])
+}
+
+func (s *Storage) UnmarshalJSON(b []byte) error {
+	var str string
+	if err := json.Unmarshal(b, &str); err != nil {
+		return err
+	}
+
+	str = strings.TrimLeft(str, "0x")
+	if len(str) == 0 {
+		return nil
+	}
+
+	if len(str)%2 == 1 {
+		str = "0" + str
+	}
+
+	val, err := hex.DecodeString(str)
+	if err != nil {
+		return err
+	}
+
+	if len(val) > 32 {
+		return fmt.Errorf("storage key/value cannot be greater than 32 bytes")
+	}
+	copy((*s)[:len(val)-1], val[:])
+	return nil
+}
+
+func (s Storage) MarshalJSON() ([]byte, error) {
+	trimmed := bytes.TrimLeft(s[:], "\x00")
+	return json.Marshal(trimmed)
+}
+
+func (s *Storage) EncodeRLP(_w io.Writer) error {
+	var obj Storage
+	if s != nil {
+		obj = *s
+	}
+	trimmed := bytes.TrimLeft(obj[:], "\x00")
+	return rlp.Encode(_w, trimmed)
+}
+
+func (s *Storage) DecodeRLP(dec *rlp.Stream) error {
+	err := dec.ReadBytes((*s)[:])
+	return err
 }
 
 // encodingStorageWrite is the encoding format of SlotWrites.
 type encodingSlotWrites struct {
-	Slot     common.Hash            `json:"slot"`
+	Slot     Storage                `json:"slot"`
 	Accesses []encodingStorageWrite `json:"accesses"`
 }
 
@@ -165,7 +221,7 @@ func (e *encodingSlotWrites) validate() error {
 type AccountAccess struct {
 	Address        common.Address          `json:"address,omitempty"`        // 20-byte Ethereum address
 	StorageChanges []encodingSlotWrites    `json:"storageChanges,omitempty"` // Storage changes (slot -> [tx_index -> new_value])
-	StorageReads   []common.Hash           `json:"storageReads,omitempty"`   // Read-only storage keys
+	StorageReads   []Storage               `json:"storageReads,omitempty"`   // Read-only storage keys
 	BalanceChanges []encodingBalanceChange `json:"balanceChanges,omitempty"` // Balance changes ([tx_index -> post_balance])
 	NonceChanges   []encodingAccountNonce  `json:"nonceChanges,omitempty"`   // Nonce changes ([tx_index -> new_nonce])
 	CodeChanges    []CodeChange            `json:"code,omitempty"`           // CodeChanges changes ([tx_index -> new_code])
@@ -186,8 +242,8 @@ func (e *AccountAccess) validate() error {
 			return err
 		}
 	}
-	readKeys := make(map[common.Hash]struct{})
+	readKeys := make(map[Storage]struct{})
-	writeKeys := make(map[common.Hash]struct{})
+	writeKeys := make(map[Storage]struct{})
 	for _, readKey := range e.StorageReads {
 		if _, ok := readKeys[readKey]; ok {
 			return errors.New("duplicate read key")
@@ -209,7 +265,7 @@ func (e *AccountAccess) validate() error {
 	}
 
 	// Check the storage read slots are sorted in order
-	if !slices.IsSortedFunc(e.StorageReads, func(a, b common.Hash) int {
+	if !slices.IsSortedFunc(e.StorageReads, func(a, b Storage) int {
 		return bytes.Compare(a[:], b[:])
 	}) {
 		return errors.New("storage read slots not in lexicographic order")
@@ -276,7 +332,7 @@ func (a *ConstructionAccountAccesses) toEncodingObj(addr common.Address) Account
 	res := AccountAccess{
 		Address:        addr,
 		StorageChanges: make([]encodingSlotWrites, 0),
-		StorageReads:   make([]common.Hash, 0),
+		StorageReads:   make([]Storage, 0),
 		BalanceChanges: make([]encodingBalanceChange, 0),
 		NonceChanges:   make([]encodingAccountNonce, 0),
 		CodeChanges:    make([]CodeChange, 0),
@@ -284,10 +340,10 @@ func (a *ConstructionAccountAccesses) toEncodingObj(addr common.Address) Account
 
 	// Convert write slots
 	writeSlots := slices.Collect(maps.Keys(a.StorageWrites))
-	slices.SortFunc(writeSlots, common.Hash.Cmp)
+	slices.SortFunc(writeSlots, Storage.Cmp)
 	for _, slot := range writeSlots {
 		var obj encodingSlotWrites
-		obj.Slot = slot
+		obj.Slot = Storage(slot)
 
 		slotWrites := a.StorageWrites[slot]
 		obj.Accesses = make([]encodingStorageWrite, 0, len(slotWrites))
@@ -297,7 +353,7 @@ func (a *ConstructionAccountAccesses) toEncodingObj(addr common.Address) Account
 		for _, index := range indices {
 			obj.Accesses = append(obj.Accesses, encodingStorageWrite{
 				TxIdx:      index,
-				ValueAfter: slotWrites[index],
+				ValueAfter: Storage(slotWrites[index]),
 			})
 		}
 		res.StorageChanges = append(res.StorageChanges, obj)
@@ -305,7 +361,7 @@ func (a *ConstructionAccountAccesses) toEncodingObj(addr common.Address) Account
 
 	// Convert read slots
 	readSlots := slices.Collect(maps.Keys(a.StorageReads))
-	slices.SortFunc(readSlots, common.Hash.Cmp)
+	slices.SortFunc(readSlots, Storage.Cmp)
 	for _, slot := range readSlots {
 		res.StorageReads = append(res.StorageReads, slot)
 	}

core/types/bal/bal_encoding_rlp_generated.go

@@ -87,7 +87,7 @@ func (obj *AccountAccess) DecodeRLP(dec *rlp.Stream) error {
 					return err
 				}
 				// Slot:
-				var _tmp4 common.Hash
+				var _tmp4 Storage
 				if err := dec.ReadBytes(_tmp4[:]); err != nil {
 					return err
 				}
@@ -110,7 +110,7 @@ func (obj *AccountAccess) DecodeRLP(dec *rlp.Stream) error {
 						}
 						_tmp6.TxIdx = _tmp7
 						// ValueAfter:
-						var _tmp8 common.Hash
+						var _tmp8 Storage
 						if err := dec.ReadBytes(_tmp8[:]); err != nil {
 							return err
 						}
@@ -136,12 +136,12 @@ func (obj *AccountAccess) DecodeRLP(dec *rlp.Stream) error {
 		}
 		_tmp0.StorageChanges = _tmp2
 		// StorageReads:
-		var _tmp9 []common.Hash
+		var _tmp9 []Storage
 		if _, err := dec.List(); err != nil {
 			return err
 		}
 		for dec.MoreDataInList() {
-			var _tmp10 common.Hash
+			var _tmp10 Storage
 			if err := dec.ReadBytes(_tmp10[:]); err != nil {
 				return err
 			}