mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-08 07:58:40 +00:00
core, eth: fix post-sync block processing and BAL type compatibility
Fix the post-sync deadlock where blocks validated via BAL in newPayload were never written to the database, causing ForkchoiceUpdated to fail finding them and triggering infinite sync cycles. Changes: - Export WriteBlockWithoutState and call it after ProcessBlockWithBAL in newPayload, so FCU can find blocks via GetBlockByHash - Guard SetCanonical against recoverAncestors for partial state nodes (they can't re-execute blocks, only apply BAL diffs) - Auto-disable log indexing when partial state is enabled (no receipts) - Fix BAL type field accesses to match upstream bal-devnet-2 types (StorageChanges, CodeChanges, BalanceChanges, Validate signature) - Update newPayload signature (BAL now comes from ExecutableData params) - Add partial sync scripts and documentation Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
CPerezz
parent
4cd7b3ba6c
commit
c3c4dfd838
15 changed files with 2747 additions and 69 deletions
|
|
@ -1811,10 +1811,10 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
// writeBlockWithoutState writes only the block and its metadata to the database,
|
||||
// WriteBlockWithoutState writes only the block and its metadata to the database,
|
||||
// but does not write any state. This is used to construct competing side forks
|
||||
// up to the point where they exceed the canonical total difficulty.
|
||||
func (bc *BlockChain) writeBlockWithoutState(block *types.Block) (err error) {
|
||||
func (bc *BlockChain) WriteBlockWithoutState(block *types.Block) (err error) {
|
||||
if bc.insertStopped() {
|
||||
return errInsertionInterrupted
|
||||
}
|
||||
|
|
@ -2622,7 +2622,7 @@ func (bc *BlockChain) insertSideChain(ctx context.Context, block *types.Block, i
|
|||
}
|
||||
if !bc.HasBlock(block.Hash(), block.NumberU64()) {
|
||||
start := time.Now()
|
||||
if err := bc.writeBlockWithoutState(block); err != nil {
|
||||
if err := bc.WriteBlockWithoutState(block); err != nil {
|
||||
return nil, it.index, err
|
||||
}
|
||||
log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(),
|
||||
|
|
@ -2982,6 +2982,11 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
|
|||
|
||||
// Re-execute the reorged chain in case the head state is missing.
|
||||
if !bc.HasState(head.Root()) {
|
||||
// Partial state nodes can't re-execute blocks — they only apply BAL diffs.
|
||||
// If state is missing here, it's an error in the partial state pipeline.
|
||||
if bc.partialState != nil {
|
||||
return common.Hash{}, fmt.Errorf("partial state: missing state for block %d root %x", head.NumberU64(), head.Root())
|
||||
}
|
||||
if latestValidHash, err := bc.recoverAncestors(context.Background(), head, false); err != nil {
|
||||
return latestValidHash, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,7 @@ func (bc *BlockChain) ProcessBlockWithBAL(
|
|||
// pre-attested by the Consensus Layer. See function documentation above.
|
||||
|
||||
// 1. Validate BAL structure
|
||||
if err := accessList.Validate(); err != nil {
|
||||
if err := accessList.Validate(len(block.Transactions())); err != nil {
|
||||
return fmt.Errorf("invalid BAL structure: %w", err)
|
||||
}
|
||||
|
||||
|
|
@ -107,7 +107,7 @@ func (bc *BlockChain) ProcessBlockWithBAL(
|
|||
"number", block.NumberU64(),
|
||||
"hash", block.Hash().Hex(),
|
||||
"root", newRoot.Hex(),
|
||||
"accounts", len(accessList.Accesses))
|
||||
"accounts", len(*accessList))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -149,9 +149,8 @@ func TestProcessBlockWithBAL_InvalidBAL(t *testing.T) {
|
|||
|
||||
// Create invalid BAL (nil Accesses slice would be valid, but we need to test validation)
|
||||
// For now, test with a valid but empty BAL to ensure the flow works
|
||||
accessList := &bal.BlockAccessList{
|
||||
Accesses: []bal.AccountAccess{},
|
||||
}
|
||||
emptyBAL := bal.BlockAccessList{}
|
||||
accessList := &emptyBAL
|
||||
|
||||
// This should fail because computed root won't match header root after applying empty BAL
|
||||
// The actual root computation depends on the parent state
|
||||
|
|
@ -182,8 +181,10 @@ func TestProcessBlockWithBAL_StateRootMismatch(t *testing.T) {
|
|||
block := types.NewBlock(header, nil, nil, nil)
|
||||
|
||||
// Create BAL that changes state
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(5000))
|
||||
cbal := make(bal.ConstructionBlockAccessList)
|
||||
cbal[addr] = &bal.ConstructionAccountAccesses{
|
||||
BalanceChanges: map[uint16]*uint256.Int{0: uint256.NewInt(5000)},
|
||||
}
|
||||
accessList := constructionToBlockAccessListCore(t, &cbal)
|
||||
|
||||
err := bc.ProcessBlockWithBAL(block, accessList)
|
||||
|
|
|
|||
|
|
@ -100,13 +100,13 @@ func (s *PartialState) ApplyBALAndComputeRoot(parentRoot common.Hash, accessList
|
|||
}
|
||||
|
||||
// Collect all account states with origin tracking
|
||||
accounts := make([]*accountState, 0, len(accessList.Accesses))
|
||||
accounts := make([]*accountState, 0, len(*accessList))
|
||||
|
||||
// Collect all trie nodes for batched update
|
||||
allNodes := trienode.NewMergedNodeSet()
|
||||
|
||||
// Phase 1: Process each account's changes from BAL
|
||||
for _, access := range accessList.Accesses {
|
||||
for _, access := range *accessList {
|
||||
addr := common.BytesToAddress(access.Address[:])
|
||||
|
||||
// Get current account state with origin tracking
|
||||
|
|
@ -151,7 +151,7 @@ func (s *PartialState) ApplyBALAndComputeRoot(parentRoot common.Hash, accessList
|
|||
// Apply balance changes (use final value from last tx)
|
||||
if len(access.BalanceChanges) > 0 {
|
||||
lastChange := access.BalanceChanges[len(access.BalanceChanges)-1]
|
||||
account.Balance = new(uint256.Int).SetBytes(lastChange.Balance[:])
|
||||
account.Balance = new(uint256.Int).Set(lastChange.Balance)
|
||||
state.modified = true
|
||||
}
|
||||
|
||||
|
|
@ -163,8 +163,8 @@ func (s *PartialState) ApplyBALAndComputeRoot(parentRoot common.Hash, accessList
|
|||
}
|
||||
|
||||
// Apply code changes
|
||||
if len(access.Code) > 0 {
|
||||
lastCode := access.Code[len(access.Code)-1]
|
||||
if len(access.CodeChanges) > 0 {
|
||||
lastCode := access.CodeChanges[len(access.CodeChanges)-1]
|
||||
codeHash := crypto.Keccak256Hash(lastCode.Code)
|
||||
account.CodeHash = codeHash.Bytes()
|
||||
state.modified = true
|
||||
|
|
@ -177,7 +177,7 @@ func (s *PartialState) ApplyBALAndComputeRoot(parentRoot common.Hash, accessList
|
|||
|
||||
// Apply storage changes (only for tracked contracts)
|
||||
// CRITICAL: Commit storage trie HERE, before account trie
|
||||
if len(access.StorageWrites) > 0 && s.filter.IsTracked(addr) {
|
||||
if len(access.StorageChanges) > 0 && s.filter.IsTracked(addr) {
|
||||
newStorageRoot, storageNodes, err := s.applyStorageChanges(
|
||||
addr, parentRoot, account.Root, &access)
|
||||
if err != nil {
|
||||
|
|
@ -276,21 +276,22 @@ func (s *PartialState) buildStateSet(accounts []*accountState, accessList *bal.B
|
|||
// addStorageToStateSet finds storage writes for the given address and adds them to the StateSet.
|
||||
func (s *PartialState) addStorageToStateSet(stateSet *triedb.StateSet, addr common.Address, addrHash common.Hash, accessList *bal.BlockAccessList) {
|
||||
// Find this account's storage writes in BAL
|
||||
for _, access := range accessList.Accesses {
|
||||
accessAddr := common.BytesToAddress(access.Address[:])
|
||||
for _, access := range *accessList {
|
||||
accessAddr := access.Address
|
||||
if accessAddr != addr {
|
||||
continue
|
||||
}
|
||||
if len(access.StorageWrites) == 0 {
|
||||
if len(access.StorageChanges) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
storageMap := make(map[common.Hash][]byte)
|
||||
for _, slotWrite := range access.StorageWrites {
|
||||
slotHash := crypto.Keccak256Hash(slotWrite.Slot[:])
|
||||
for _, slotWrite := range access.StorageChanges {
|
||||
slotKey := slotWrite.Slot.ToHash()
|
||||
slotHash := crypto.Keccak256Hash(slotKey[:])
|
||||
if len(slotWrite.Accesses) > 0 {
|
||||
lastWrite := slotWrite.Accesses[len(slotWrite.Accesses)-1]
|
||||
value := common.BytesToHash(lastWrite.ValueAfter[:])
|
||||
value := lastWrite.ValueAfter.ToHash()
|
||||
if value == (common.Hash{}) {
|
||||
storageMap[slotHash] = nil // nil = deletion
|
||||
} else {
|
||||
|
|
@ -332,15 +333,15 @@ func (s *PartialState) applyStorageChanges(
|
|||
}
|
||||
|
||||
// Apply each storage write (use final value)
|
||||
for _, slotWrite := range access.StorageWrites {
|
||||
slot := common.BytesToHash(slotWrite.Slot[:])
|
||||
for _, slotWrite := range access.StorageChanges {
|
||||
slot := slotWrite.Slot.ToHash()
|
||||
|
||||
// Get final value (last write wins)
|
||||
if len(slotWrite.Accesses) == 0 {
|
||||
continue
|
||||
}
|
||||
lastWrite := slotWrite.Accesses[len(slotWrite.Accesses)-1]
|
||||
value := common.BytesToHash(lastWrite.ValueAfter[:])
|
||||
value := lastWrite.ValueAfter.ToHash()
|
||||
|
||||
if value == (common.Hash{}) {
|
||||
// Delete slot
|
||||
|
|
|
|||
|
|
@ -32,13 +32,67 @@ import (
|
|||
"github.com/holiman/uint256"
|
||||
)
|
||||
|
||||
// constructionToBlockAccessList converts ConstructionBlockAccessList to BlockAccessList
|
||||
// via RLP encoding/decoding.
|
||||
func constructionToBlockAccessList(t *testing.T, cbal *bal.ConstructionBlockAccessList) *bal.BlockAccessList {
|
||||
// testBALBuilder is a test helper for constructing BlockAccessLists.
|
||||
// It wraps ConstructionBlockAccessList and provides convenience methods
|
||||
// matching the test patterns (BalanceChange, NonceChange, StorageWrite, CodeChange).
|
||||
type testBALBuilder struct {
|
||||
accesses bal.ConstructionBlockAccessList
|
||||
}
|
||||
|
||||
func newTestBALBuilder() *testBALBuilder {
|
||||
return &testBALBuilder{
|
||||
accesses: make(bal.ConstructionBlockAccessList),
|
||||
}
|
||||
}
|
||||
|
||||
func (b *testBALBuilder) ensureAccount(addr common.Address) *bal.ConstructionAccountAccesses {
|
||||
if _, ok := b.accesses[addr]; !ok {
|
||||
b.accesses[addr] = &bal.ConstructionAccountAccesses{}
|
||||
}
|
||||
return b.accesses[addr]
|
||||
}
|
||||
|
||||
func (b *testBALBuilder) BalanceChange(txIdx uint16, addr common.Address, balance *uint256.Int) {
|
||||
acc := b.ensureAccount(addr)
|
||||
if acc.BalanceChanges == nil {
|
||||
acc.BalanceChanges = make(map[uint16]*uint256.Int)
|
||||
}
|
||||
acc.BalanceChanges[txIdx] = balance
|
||||
}
|
||||
|
||||
func (b *testBALBuilder) NonceChange(addr common.Address, txIdx uint16, nonce uint64) {
|
||||
acc := b.ensureAccount(addr)
|
||||
if acc.NonceChanges == nil {
|
||||
acc.NonceChanges = make(map[uint16]uint64)
|
||||
}
|
||||
acc.NonceChanges[txIdx] = nonce
|
||||
}
|
||||
|
||||
func (b *testBALBuilder) StorageWrite(txIdx uint16, addr common.Address, slot, value common.Hash) {
|
||||
acc := b.ensureAccount(addr)
|
||||
if acc.StorageWrites == nil {
|
||||
acc.StorageWrites = make(map[common.Hash]map[uint16]common.Hash)
|
||||
}
|
||||
if _, ok := acc.StorageWrites[slot]; !ok {
|
||||
acc.StorageWrites[slot] = make(map[uint16]common.Hash)
|
||||
}
|
||||
acc.StorageWrites[slot][txIdx] = value
|
||||
}
|
||||
|
||||
func (b *testBALBuilder) CodeChange(addr common.Address, txIdx uint16, code []byte) {
|
||||
acc := b.ensureAccount(addr)
|
||||
if acc.CodeChanges == nil {
|
||||
acc.CodeChanges = make(map[uint16]bal.CodeChange)
|
||||
}
|
||||
acc.CodeChanges[txIdx] = bal.CodeChange{TxIdx: txIdx, Code: code}
|
||||
}
|
||||
|
||||
// Build converts the construction BAL to the encoding format via RLP round-trip.
|
||||
func (b *testBALBuilder) Build(t *testing.T) *bal.BlockAccessList {
|
||||
t.Helper()
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := cbal.EncodeRLP(&buf); err != nil {
|
||||
if err := b.accesses.EncodeRLP(&buf); err != nil {
|
||||
t.Fatalf("failed to encode BAL: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -100,9 +154,8 @@ func TestApplyBALAndComputeRoot_EmptyBAL(t *testing.T) {
|
|||
ps, _, emptyRoot := setupTestPartialState(t, nil)
|
||||
|
||||
// Apply empty BAL
|
||||
accessList := &bal.BlockAccessList{
|
||||
Accesses: []bal.AccountAccess{},
|
||||
}
|
||||
emptyBAL := bal.BlockAccessList{}
|
||||
accessList := &emptyBAL
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(emptyRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -131,10 +184,10 @@ func TestApplyBALAndComputeRoot_BalanceChange(t *testing.T) {
|
|||
|
||||
// Create BAL with balance change using ConstructionBlockAccessList
|
||||
newBalance := uint256.NewInt(2000)
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, newBalance)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -174,10 +227,10 @@ func TestApplyBALAndComputeRoot_NonceChange(t *testing.T) {
|
|||
parentRoot := setupTestStateWithAccount(t, trieDB, addr, initialAccount)
|
||||
|
||||
// Create BAL with nonce change
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.NonceChange(addr, 0, 6)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -215,10 +268,10 @@ func TestApplyBALAndComputeRoot_StorageChange(t *testing.T) {
|
|||
slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001")
|
||||
value := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000042")
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.StorageWrite(0, addr, slot, value)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -274,11 +327,11 @@ func TestApplyBALAndComputeRoot_UntrackedContractStorageIgnored(t *testing.T) {
|
|||
slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001")
|
||||
value := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000042")
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.StorageWrite(0, trackedAddr, slot, value)
|
||||
cbal.StorageWrite(0, untrackedAddr, slot, value)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -306,11 +359,11 @@ func TestApplyBALAndComputeRoot_NewAccount(t *testing.T) {
|
|||
// Create BAL that creates a new account
|
||||
balance := uint256.NewInt(1000)
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, balance)
|
||||
cbal.NonceChange(addr, 0, 1)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(emptyRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -354,10 +407,10 @@ func TestApplyBALAndComputeRoot_CodeChange(t *testing.T) {
|
|||
code := []byte{0x60, 0x60, 0x60, 0x40, 0x52} // Some bytecode
|
||||
codeHash := crypto.Keccak256Hash(code)
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.CodeChange(addr, 0, code)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -396,7 +449,7 @@ func TestApplyBALAndComputeRoot_MultipleTransactions(t *testing.T) {
|
|||
balance2 := uint256.NewInt(2000)
|
||||
balance3 := uint256.NewInt(1500) // Final balance
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, balance1)
|
||||
cbal.BalanceChange(1, addr, balance2)
|
||||
cbal.BalanceChange(2, addr, balance3) // Final
|
||||
|
|
@ -404,7 +457,7 @@ func TestApplyBALAndComputeRoot_MultipleTransactions(t *testing.T) {
|
|||
cbal.NonceChange(addr, 1, 2)
|
||||
cbal.NonceChange(addr, 2, 3) // Final
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -469,10 +522,10 @@ func TestApplyBALAndComputeRoot_StorageDeletion(t *testing.T) {
|
|||
trieDB.Commit(parentRoot, false)
|
||||
|
||||
// Create BAL that deletes the storage slot (write zero value)
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.StorageWrite(0, addr, slot, common.Hash{}) // Zero value = delete
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -507,12 +560,12 @@ func TestApplyBALAndComputeRoot_MultipleStorageWritesSameSlot(t *testing.T) {
|
|||
value2 := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000002")
|
||||
value3 := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000003") // Final
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.StorageWrite(0, addr, slot, value1)
|
||||
cbal.StorageWrite(1, addr, slot, value2)
|
||||
cbal.StorageWrite(2, addr, slot, value3) // Final value
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -555,10 +608,10 @@ func TestApplyBALAndComputeRoot_AccountDeletion_EIP161(t *testing.T) {
|
|||
parentRoot := setupTestStateWithAccount(t, trieDB, addr, initialAccount)
|
||||
|
||||
// Create BAL that empties the account
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(0)) // Zero balance
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -584,10 +637,10 @@ func TestApplyBALAndComputeRoot_NeverExistedEmptyAccount(t *testing.T) {
|
|||
|
||||
// Create BAL that "touches" an account but leaves it empty
|
||||
// This simulates an account that receives 0 balance and sends 0 balance
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(0)) // Zero balance on never-existed account
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(emptyRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -641,11 +694,11 @@ func TestApplyBALAndComputeRoot_CodeChangeUntracked(t *testing.T) {
|
|||
code := []byte{0x60, 0x60, 0x60, 0x40, 0x52}
|
||||
codeHash := crypto.Keccak256Hash(code)
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.CodeChange(trackedAddr, 0, code)
|
||||
cbal.CodeChange(untrackedAddr, 0, code)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -701,13 +754,13 @@ func TestApplyBALAndComputeRoot_MixedChanges(t *testing.T) {
|
|||
slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001")
|
||||
value := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000042")
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, newBalance)
|
||||
cbal.NonceChange(addr, 0, newNonce)
|
||||
cbal.CodeChange(addr, 0, code)
|
||||
cbal.StorageWrite(0, addr, slot, value)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -752,9 +805,9 @@ func TestApplyBALAndComputeRoot_ErrorInvalidParentRoot(t *testing.T) {
|
|||
// Use a non-existent root
|
||||
invalidRoot := common.HexToHash("0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef")
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(1000))
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
_, err := ps.ApplyBALAndComputeRoot(invalidRoot, accessList)
|
||||
if err == nil {
|
||||
|
|
@ -871,9 +924,9 @@ func TestBuildStateSet_AccountModification(t *testing.T) {
|
|||
parentRoot := setupTestStateWithAccount(t, trieDB, addr, initialAccount)
|
||||
|
||||
// Apply balance change
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(2000))
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -916,9 +969,9 @@ func TestBuildStateSet_StorageRLPEncoding(t *testing.T) {
|
|||
slot := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001")
|
||||
value := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000042")
|
||||
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.StorageWrite(0, addr, slot, value)
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -961,10 +1014,10 @@ func TestBuildStateSet_OriginTracking(t *testing.T) {
|
|||
parentRoot := setupTestStateWithAccount(t, trieDB, addr, initialAccount)
|
||||
|
||||
// Modify the account
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
cbal.BalanceChange(0, addr, uint256.NewInt(6000))
|
||||
cbal.NonceChange(addr, 0, 11)
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
@ -1023,7 +1076,7 @@ func TestApplyBALAndComputeRoot_MultipleAccountTypes(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create BAL with different changes for each account
|
||||
cbal := bal.NewConstructionBlockAccessList()
|
||||
cbal := newTestBALBuilder()
|
||||
|
||||
// addr1: balance change
|
||||
cbal.BalanceChange(0, addr1, uint256.NewInt(2000))
|
||||
|
|
@ -1037,7 +1090,7 @@ func TestApplyBALAndComputeRoot_MultipleAccountTypes(t *testing.T) {
|
|||
cbal.BalanceChange(0, addr3, uint256.NewInt(3000))
|
||||
cbal.NonceChange(addr3, 0, 1)
|
||||
|
||||
accessList := constructionToBlockAccessList(t, &cbal)
|
||||
accessList := cbal.Build(t)
|
||||
|
||||
newRoot, err := ps.ApplyBALAndComputeRoot(parentRoot, accessList)
|
||||
if err != nil {
|
||||
|
|
|
|||
185
docs/partial-state/DEVNET_TESTING.md
Normal file
185
docs/partial-state/DEVNET_TESTING.md
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
# Partial State Devnet Testing Guide
|
||||
|
||||
This document describes how to test partial statefulness with a local devnet using 2 geth instances.
|
||||
|
||||
## Overview
|
||||
|
||||
Partial state nodes:
|
||||
- Sync all account data (balances, nonces, code hashes)
|
||||
- Only store storage for tracked contracts
|
||||
- Process blocks using BAL (Block Access Lists) instead of re-executing transactions
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Go 1.22+ installed
|
||||
- Two terminal windows
|
||||
- Build geth with partial state support:
|
||||
```bash
|
||||
go build ./cmd/geth
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
### Terminal 1: Full Node (creates blocks in dev mode)
|
||||
|
||||
```bash
|
||||
# Create fresh data directory
|
||||
rm -rf /tmp/full-node
|
||||
|
||||
# Start full node in dev mode
|
||||
./geth --datadir /tmp/full-node \
|
||||
--dev \
|
||||
--dev.period 5 \
|
||||
--port 30303 \
|
||||
--http --http.port 8545 \
|
||||
--http.api eth,net,web3,debug,admin \
|
||||
--verbosity 3
|
||||
|
||||
# Get the enode URL (run in another terminal or use geth attach)
|
||||
# geth attach /tmp/full-node/geth.ipc --exec admin.nodeInfo.enode
|
||||
```
|
||||
|
||||
### Terminal 2: Partial State Node (receives blocks via P2P)
|
||||
|
||||
First, get the enode from the full node:
|
||||
```bash
|
||||
ENODE=$(geth attach /tmp/full-node/geth.ipc --exec admin.nodeInfo.enode | tr -d '"')
|
||||
echo "Full node enode: $ENODE"
|
||||
```
|
||||
|
||||
Then start the partial state node:
|
||||
```bash
|
||||
# Create fresh data directory
|
||||
rm -rf /tmp/partial-node
|
||||
|
||||
# Start partial state node
|
||||
./geth --datadir /tmp/partial-node \
|
||||
--port 30304 \
|
||||
--http --http.port 8546 \
|
||||
--http.api eth,net,web3,debug \
|
||||
--partial-state \
|
||||
--partial-state.contracts 0xContractAddr1,0xContractAddr2 \
|
||||
--bootnodes "$ENODE" \
|
||||
--networkid 1337 \
|
||||
--verbosity 3
|
||||
```
|
||||
|
||||
Note: Replace `0xContractAddr1,0xContractAddr2` with actual contract addresses you want to track.
|
||||
|
||||
## Test Scenarios
|
||||
|
||||
### 1. Block Sync Test
|
||||
|
||||
Send a transaction on the full node and verify the partial node receives it:
|
||||
|
||||
```bash
|
||||
# On full node (Terminal 1 or new terminal)
|
||||
geth attach /tmp/full-node/geth.ipc
|
||||
|
||||
# In geth console, send a transaction
|
||||
> eth.sendTransaction({from: eth.coinbase, to: "0x1234567890123456789012345678901234567890", value: web3.toWei(1, "ether")})
|
||||
|
||||
# Check block number
|
||||
> eth.blockNumber
|
||||
```
|
||||
|
||||
Verify on partial node:
|
||||
```bash
|
||||
curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8546 | jq
|
||||
```
|
||||
|
||||
### 2. Balance Query Test
|
||||
|
||||
Both nodes should return the same balance for any account:
|
||||
|
||||
```bash
|
||||
# Full node
|
||||
curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x1234567890123456789012345678901234567890","latest"],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8545 | jq
|
||||
|
||||
# Partial node
|
||||
curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x1234567890123456789012345678901234567890","latest"],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8546 | jq
|
||||
```
|
||||
|
||||
### 3. Storage Query Test
|
||||
|
||||
Deploy a contract and test storage access:
|
||||
|
||||
```bash
|
||||
# Query tracked contract storage (should work)
|
||||
curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getStorageAt","params":["0xTrackedContractAddr","0x0","latest"],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8546 | jq
|
||||
|
||||
# Query untracked contract storage (should fail or return empty)
|
||||
curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getStorageAt","params":["0xUntrackedContractAddr","0x0","latest"],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8546 | jq
|
||||
```
|
||||
|
||||
### 4. State Root Verification
|
||||
|
||||
Verify both nodes have the same state root:
|
||||
|
||||
```bash
|
||||
# Get latest block from both nodes
|
||||
FULL_ROOT=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8545 | jq -r '.result.stateRoot')
|
||||
|
||||
PARTIAL_ROOT=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["latest",false],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:8546 | jq -r '.result.stateRoot')
|
||||
|
||||
echo "Full node state root: $FULL_ROOT"
|
||||
echo "Partial node state root: $PARTIAL_ROOT"
|
||||
|
||||
if [ "$FULL_ROOT" = "$PARTIAL_ROOT" ]; then
|
||||
echo "State roots match!"
|
||||
else
|
||||
echo "State roots DO NOT match!"
|
||||
fi
|
||||
```
|
||||
|
||||
## Database Size Comparison
|
||||
|
||||
After syncing, compare database sizes:
|
||||
|
||||
```bash
|
||||
echo "Full node database size:"
|
||||
du -sh /tmp/full-node/geth/chaindata
|
||||
|
||||
echo "Partial node database size:"
|
||||
du -sh /tmp/partial-node/geth/chaindata
|
||||
```
|
||||
|
||||
The partial node should have a significantly smaller database size due to skipped storage.
|
||||
|
||||
## Cleanup
|
||||
|
||||
```bash
|
||||
# Stop both geth instances (Ctrl+C in each terminal)
|
||||
|
||||
# Remove test data
|
||||
rm -rf /tmp/full-node /tmp/partial-node
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Nodes not connecting
|
||||
- Verify bootnodes enode URL is correct
|
||||
- Check that network IDs match (dev mode uses 1337)
|
||||
- Ensure ports are not blocked
|
||||
|
||||
### State root mismatch
|
||||
- This indicates a bug in BAL processing
|
||||
- Check geth logs for errors during block processing
|
||||
- Verify the partial node received the BAL with the block
|
||||
|
||||
### Storage queries failing
|
||||
- Verify the contract address is in the tracked contracts list
|
||||
- Check that the contract was deployed after the partial node started syncing
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [EIP-7928: Block Access Lists](https://eips.ethereum.org/EIPS/eip-7928)
|
||||
- [Partial Statefulness Master Plan](./PARTIAL_STATEFULNESS_PLAN.md)
|
||||
- [Phase 3 Implementation Plan](./PHASE3_PLAN.md)
|
||||
543
docs/partial-state/PARTIAL_STATEFULNESS_PLAN.md
Normal file
543
docs/partial-state/PARTIAL_STATEFULNESS_PLAN.md
Normal file
|
|
@ -0,0 +1,543 @@
|
|||
# Partial Statefulness Design - Final Plan
|
||||
|
||||
## Overview
|
||||
|
||||
**Goal**: Enable Ethereum nodes to operate with reduced storage by keeping:
|
||||
- Full account trie (all accounts + intermediate nodes)
|
||||
- Selective storage (only configured contracts' storage)
|
||||
- BAL-based state updates (per EIP-7928)
|
||||
|
||||
**Source**: [ethresear.ch - Partial Statefulness](https://ethresear.ch/t/the-future-of-state-part-2-beyond-the-myth-of-partial-statefulness-the-reality-of-zkevms/23396)
|
||||
|
||||
---
|
||||
|
||||
## Design Decisions (Confirmed)
|
||||
|
||||
### Core Model
|
||||
| Decision | Choice | Notes |
|
||||
|----------|--------|-------|
|
||||
| Account trie | ALL accounts + ALL intermediate nodes | Full trie structure with compression |
|
||||
| Storage | Only configured contracts | User specifies which contracts in config file |
|
||||
| BAL source | Per EIP-7928 | BALs come with blocks, hash committed in header |
|
||||
| Validation | Trust BAL, apply diffs | Same trust model as light clients (signing committee) |
|
||||
| Block history | 256-1024 blocks | Support BLOCKHASH opcode, configurable BAL retention |
|
||||
|
||||
### Storage Approach
|
||||
| Component | Size | Notes |
|
||||
|-----------|------|-------|
|
||||
| Account leaves | ~14 GB | 300M accounts × ~45 bytes (slim RLP) |
|
||||
| Intermediate nodes | ~15-25 GB | With delta encoding + bitmap compression |
|
||||
| **Total account trie** | **~30-40 GB** | |
|
||||
| Configured storage | Variable | Depends on tracked contracts |
|
||||
| BAL history | ~1-2 GB | 256-1024 blocks |
|
||||
|
||||
### Operations
|
||||
| Operation | Approach |
|
||||
|-----------|----------|
|
||||
| Initial sync | Account trie first (snap sync), then configured storage |
|
||||
| Block processing | Apply BAL diffs → update trie → verify state root matches header |
|
||||
| Reorgs | Revert using stored BAL history; deeper reorgs request from full peers |
|
||||
| eth_getProof (accounts) | Supported for ALL accounts |
|
||||
| eth_getProof (storage) | Only for configured contracts; error otherwise |
|
||||
| Mempool validation | Fully supported (only needs account data) |
|
||||
| Serving peers | Account proofs + tracked contract storage |
|
||||
|
||||
---
|
||||
|
||||
## EIP-7928 BAL Integration
|
||||
|
||||
### BAL Format (from EIP-7928)
|
||||
```
|
||||
BlockAccessList = [AccountAccess, ...]
|
||||
|
||||
AccountAccess = [
|
||||
Address,
|
||||
StorageWrites, // map[slot] -> map[txIdx] -> value
|
||||
StorageReads, // list of read slots
|
||||
BalanceChanges, // map[txIdx] -> balance
|
||||
NonceChanges, // map[txIdx] -> nonce
|
||||
CodeChanges // map[txIdx] -> bytecode
|
||||
]
|
||||
```
|
||||
|
||||
### Key EIP-7928 Facts
|
||||
- **Header commitment**: `block_access_list_hash = keccak256(rlp.encode(bal))`
|
||||
- **Propagation**: Via Engine API (ExecutionPayloadV4), not in block body
|
||||
- **Retention**: Full nodes must keep WSP (~5 months); partial nodes: configurable (256-1024 blocks)
|
||||
- **Validation**: Deterministic - wrong BAL = wrong header hash = invalid block
|
||||
|
||||
### BAL Processing Flow
|
||||
```
|
||||
1. Receive block + BAL via Engine API
|
||||
2. Verify: keccak256(rlp.encode(bal)) == header.block_access_list_hash
|
||||
3. For each AccountAccess in BAL:
|
||||
a. Load current account from trie
|
||||
b. Apply balance/nonce changes (final values per block)
|
||||
c. Apply storage root update (from BAL storage writes for tracked contracts)
|
||||
d. Update account in trie
|
||||
4. Commit trie changes
|
||||
5. Verify: trie.Root() == header.stateRoot
|
||||
6. If mismatch: reject block (consensus failure elsewhere)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## State Root Verification
|
||||
|
||||
### How It Works Without Re-execution
|
||||
|
||||
Partial nodes can verify state root because:
|
||||
|
||||
1. **Full account trie stored**: All intermediate nodes available
|
||||
2. **BAL provides final values**: Post-block account state (not deltas)
|
||||
3. **Trie update is deterministic**: Same inputs → same output
|
||||
4. **Cross-check with header**: header.stateRoot must match computed root
|
||||
|
||||
### Trust Model
|
||||
|
||||
Same as beacon chain light clients:
|
||||
- Trust signing committee (attestations)
|
||||
- Verify header commitments (state root, BAL hash)
|
||||
- Detect inconsistencies via hash mismatches
|
||||
|
||||
If BAL is incorrect:
|
||||
- State root won't match → block rejected
|
||||
- Fork choice rejects the block
|
||||
- Partial node follows canonical chain
|
||||
|
||||
---
|
||||
|
||||
## Snap Sync Adaptation
|
||||
|
||||
### Current Snap Sync (Full Node)
|
||||
```
|
||||
Phase 1: Sync account ranges (GetAccountRangeMsg)
|
||||
Phase 2: Sync all storage for all contracts
|
||||
Phase 3: Sync all bytecode
|
||||
Phase 4: Healing (fill gaps)
|
||||
```
|
||||
|
||||
### Partial Statefulness Snap Sync
|
||||
```
|
||||
Phase 1: Sync COMPLETE account trie (same as full node)
|
||||
- All accounts
|
||||
- All intermediate nodes
|
||||
- ~30-40 GB
|
||||
|
||||
Phase 2: Sync storage ONLY for configured contracts
|
||||
- Filter: Only request storage for contracts in config
|
||||
- Skip: All other contracts' storage
|
||||
|
||||
Phase 3: Sync bytecode ONLY for configured contracts
|
||||
- Same filtering as storage
|
||||
|
||||
Phase 4: Healing (account trie only)
|
||||
- No healing needed for skipped storage
|
||||
```
|
||||
|
||||
### Implementation Changes Needed
|
||||
1. Add `PartialStateConfig` to ethconfig
|
||||
2. Modify `storageRequest` creation in snap syncer to check config
|
||||
3. Skip storage/bytecode tasks for non-configured contracts
|
||||
4. Track sync progress separately for account trie vs. storage
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Config Structure
|
||||
```go
|
||||
type PartialStateConfig struct {
|
||||
Enabled bool
|
||||
Contracts []common.Address // Tracked contracts
|
||||
ContractsFile string // Or load from JSON file
|
||||
BALRetention uint64 // Blocks to keep (default: 256)
|
||||
}
|
||||
```
|
||||
|
||||
### Example Config (TOML)
|
||||
```toml
|
||||
[Eth.PartialState]
|
||||
Enabled = true
|
||||
BALRetention = 256
|
||||
Contracts = [
|
||||
"0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", # WETH
|
||||
"0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48", # USDC
|
||||
]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## RPC Behavior
|
||||
|
||||
| Method | Behavior |
|
||||
|--------|----------|
|
||||
| `eth_getBalance` | ✅ Works (have account data) |
|
||||
| `eth_getTransactionCount` | ✅ Works (have nonce) |
|
||||
| `eth_getCode` | ✅ For tracked contracts; ❌ error for others |
|
||||
| `eth_getStorageAt` | ✅ For tracked contracts; ❌ error for others |
|
||||
| `eth_getProof` (account) | ✅ Works for ANY account |
|
||||
| `eth_getProof` (storage) | ✅ For tracked contracts; ❌ error for others |
|
||||
| `eth_call` | ✅ If touches only tracked contracts; ❌ if touches untracked |
|
||||
| `eth_estimateGas` | Same as eth_call |
|
||||
| `eth_sendRawTransaction` | ✅ Mempool validation works (only needs account data) |
|
||||
|
||||
---
|
||||
|
||||
## Binary Trie (EIP-7864) Compatibility
|
||||
|
||||
### Will This Design Work With Binary Trie?
|
||||
|
||||
**Yes**, with minimal changes:
|
||||
|
||||
| Aspect | MPT | Binary Trie | Compatibility |
|
||||
|--------|-----|-------------|---------------|
|
||||
| Account data | StateAccount struct | Same struct | ✅ Compatible |
|
||||
| Trie interface | `Trie` interface | Same interface | ✅ Compatible |
|
||||
| BAL format | Per EIP-7928 | Same format | ✅ Compatible |
|
||||
| Selective storage | Skip storage tries | Skip stem suffixes | ✅ Compatible |
|
||||
| Proof generation | Merkle proofs | Path proofs | ✅ Use interface |
|
||||
|
||||
### Adaptation Needed
|
||||
Only the storage size estimates change:
|
||||
- Binary Trie total: ~48 GB (vs. MPT ~30-40 GB with compression)
|
||||
- Binary Trie has simpler structure, no compression needed
|
||||
|
||||
**Recommendation**: Use go-ethereum's `Trie` interface which abstracts over both.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Configuration & Infrastructure
|
||||
- Add `PartialStateConfig` to `eth/ethconfig/config.go`
|
||||
- Create `core/state/partial/` package with `ContractFilter` interface
|
||||
- Add CLI flags for partial state mode
|
||||
|
||||
### Phase 2: Snap Sync Modifications
|
||||
- Modify `eth/protocols/snap/sync.go` for selective storage sync
|
||||
- Add filter checks in `processAccountResponse` and `processStorageResponse`
|
||||
- Track separate progress for account trie vs. storage
|
||||
|
||||
### Phase 3: BAL Processing
|
||||
- Implement BAL diff application in block import pipeline
|
||||
- Modify `core/blockchain.go` to use BAL for state updates
|
||||
- Add state root verification without re-execution
|
||||
|
||||
### Phase 4: RPC & Operations
|
||||
- Modify `internal/ethapi/api.go` for partial state awareness
|
||||
- Add appropriate errors for untracked contract queries
|
||||
- Implement BAL history management and reorg handling
|
||||
|
||||
---
|
||||
|
||||
## Key Files to Modify
|
||||
|
||||
| File | Changes |
|
||||
|------|---------|
|
||||
| `eth/ethconfig/config.go` | Add `PartialStateConfig` |
|
||||
| `core/state/partial/filter.go` | New: `ContractFilter` interface |
|
||||
| `eth/protocols/snap/sync.go` | Filter storage sync by config |
|
||||
| `core/blockchain.go` | BAL-based state updates |
|
||||
| `internal/ethapi/api.go` | Partial state RPC handling |
|
||||
| `cmd/utils/flags.go` | CLI flags for partial state |
|
||||
|
||||
---
|
||||
|
||||
## Open Items for Implementation
|
||||
|
||||
1. **BLOCKHASH opcode**: Verify 256 blocks of history is sufficient; check if other opcodes need block history
|
||||
|
||||
2. **Storage root verification**: When applying BAL storage diffs for tracked contracts, verify computed storage root matches account's storageRoot field
|
||||
|
||||
3. **Compression implementation**: Implement delta encoding + bitmap optimization for intermediate nodes (existing pathdb patterns can be adapted)
|
||||
|
||||
4. **Selective snap sync protocol**: Research if snap protocol needs extension or if filtering can be done client-side
|
||||
|
||||
---
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
After implementation, verify:
|
||||
- [ ] Can sync account trie completely via snap sync
|
||||
- [ ] Can sync only configured contracts' storage
|
||||
- [ ] BAL diffs apply correctly, state root matches header
|
||||
- [ ] eth_getProof works for any account (proof generation)
|
||||
- [ ] eth_getProof returns error for untracked storage
|
||||
- [ ] Mempool accepts/validates transactions correctly
|
||||
- [ ] Reorgs up to BAL retention depth work
|
||||
- [ ] Deeper reorgs trigger recovery from full peers
|
||||
- [ ] Total storage matches estimates (~30-40 GB + configured storage)
|
||||
|
||||
---
|
||||
|
||||
# DETAILED SPECIFICATIONS
|
||||
|
||||
---
|
||||
|
||||
## SPEC 1: Snap Sync Refactoring for Selective Storage
|
||||
|
||||
### Overview
|
||||
|
||||
The snap sync protocol in go-ethereum downloads account data and contract storage in parallel. For partial statefulness, we need to:
|
||||
1. Download ALL accounts (unchanged behavior)
|
||||
2. Download storage ONLY for configured contracts (new filtering)
|
||||
3. Download bytecode ONLY for configured contracts (new filtering)
|
||||
|
||||
**Design Principle**: Keep original `Syncer` implementation untouched. Create a separate syncer implementation using a strategy/interface pattern that allows selection at runtime.
|
||||
|
||||
### Architecture: Strategy Pattern
|
||||
|
||||
```
|
||||
┌─────────────────────┐
|
||||
│ SyncStrategy │ (interface)
|
||||
│ interface │
|
||||
└─────────┬───────────┘
|
||||
│
|
||||
┌───────────────┼───────────────┐
|
||||
│ │ │
|
||||
┌─────────▼─────┐ ┌──────▼──────┐ ┌─────▼───────┐
|
||||
│ FullSyncer │ │PartialSyncer│ │ (future) │
|
||||
│ (wraps orig) │ │(new impl) │ │ │
|
||||
└───────────────┘ └─────────────┘ └─────────────┘
|
||||
```
|
||||
|
||||
### Key Files
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `eth/protocols/snap/sync.go` | **UNCHANGED** - Original Syncer |
|
||||
| `eth/protocols/snap/strategy.go` | **NEW** - SyncStrategy interface |
|
||||
| `eth/protocols/snap/partial_sync.go` | **NEW** - PartialSyncer implementation |
|
||||
| `core/state/partial/filter.go` | **NEW** - ContractFilter interface |
|
||||
| `eth/downloader/downloader.go` | **MODIFIED** - Strategy selection |
|
||||
|
||||
---
|
||||
|
||||
## SPEC 2: Compression + Root Recomputation
|
||||
|
||||
### Overview
|
||||
|
||||
For partial statefulness, we store the full account trie (~300M accounts + intermediate nodes) but need efficient storage. This spec covers:
|
||||
1. **REUSE** existing delta encoding infrastructure from pathdb
|
||||
2. State root recomputation from BAL diffs
|
||||
|
||||
### Existing Compression Infrastructure (REUSE - DO NOT REIMPLEMENT)
|
||||
|
||||
**Location**: `triedb/pathdb/nodes.go` (lines 431-691)
|
||||
|
||||
go-ethereum **already has production-grade compression** we must reuse:
|
||||
|
||||
| Function | Purpose | Status |
|
||||
|----------|---------|--------|
|
||||
| `encodeNodeCompressed()` | Delta encoding with bitmap | **REUSE** |
|
||||
| `decodeNodeCompressed()` | Decode compressed format | **REUSE** |
|
||||
| `encodeNodeFull()` | Full-value encoding | **REUSE** |
|
||||
| `encodeNodeHistory()` | Checkpoint + delta chains | **REUSE** |
|
||||
|
||||
---
|
||||
|
||||
## SPEC 3: BAL Processing Pipeline
|
||||
|
||||
### Overview
|
||||
|
||||
Block Access Lists (BALs) per EIP-7928 provide state diffs that allow partial nodes to update state without re-executing transactions.
|
||||
|
||||
### Existing BAL Implementation (Already in Geth)
|
||||
|
||||
**Location**: `core/types/bal/`
|
||||
|
||||
BAL types are already implemented in go-ethereum master:
|
||||
|
||||
| File | Contents |
|
||||
|------|----------|
|
||||
| `bal.go` | `ConstructionBlockAccessList`, `ConstructionAccountAccess`, builder methods |
|
||||
| `bal_encoding.go` | `BlockAccessList`, `AccountAccess`, RLP encoding, hash computation |
|
||||
| `bal_encoding_rlp_generated.go` | Generated RLP encoder/decoder |
|
||||
|
||||
---
|
||||
|
||||
## SPEC 4: RPC Modifications
|
||||
|
||||
### Overview
|
||||
|
||||
Partial state nodes can answer some RPC queries but not others. This spec defines the behavior.
|
||||
|
||||
### Error Codes
|
||||
|
||||
```go
|
||||
var (
|
||||
ErrStorageNotTracked = errors.New("storage not tracked for this contract")
|
||||
ErrCodeNotTracked = errors.New("code not tracked for this contract")
|
||||
)
|
||||
|
||||
const (
|
||||
ErrCodeStorageNotTracked = -32001
|
||||
ErrCodeNotTracked = -32002
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SPEC 5: Configuration System
|
||||
|
||||
### CLI Flags
|
||||
|
||||
```go
|
||||
var (
|
||||
PartialStateFlag = &cli.BoolFlag{
|
||||
Name: "partial-state",
|
||||
Usage: "Enable partial statefulness mode (reduced storage)",
|
||||
Category: flags.EthCategory,
|
||||
}
|
||||
|
||||
PartialStateContractsFlag = &cli.StringSliceFlag{
|
||||
Name: "partial-state.contracts",
|
||||
Usage: "Contracts to track storage for (comma-separated addresses)",
|
||||
Category: flags.EthCategory,
|
||||
}
|
||||
|
||||
PartialStateContractsFileFlag = &cli.StringFlag{
|
||||
Name: "partial-state.contracts-file",
|
||||
Usage: "JSON file containing contracts to track",
|
||||
Category: flags.EthCategory,
|
||||
}
|
||||
|
||||
PartialStateBALRetentionFlag = &cli.Uint64Flag{
|
||||
Name: "partial-state.bal-retention",
|
||||
Usage: "Number of blocks to retain BAL history (default: 256)",
|
||||
Value: 256,
|
||||
Category: flags.EthCategory,
|
||||
}
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Implementation Task Breakdown
|
||||
|
||||
### Phase 1: Core Infrastructure (Foundation)
|
||||
|
||||
| Task ID | Task | Dependencies | Effort |
|
||||
|---------|------|--------------|--------|
|
||||
| 1.1 | Create `core/state/partial/` package structure | None | S |
|
||||
| 1.2 | Implement `ContractFilter` interface | 1.1 | S |
|
||||
| 1.3 | Add `PartialStateConfig` to ethconfig | None | S |
|
||||
| 1.4 | Add CLI flags for partial state | 1.3 | S |
|
||||
| 1.5 | Implement config loading (file + direct) | 1.3, 1.4 | M |
|
||||
|
||||
### Phase 2: Snap Sync Modifications (Selective Sync via Strategy Pattern)
|
||||
|
||||
| Task ID | Task | Dependencies | Effort |
|
||||
|---------|------|--------------|--------|
|
||||
| 2.1 | Create `SyncStrategy` interface in `strategy.go` | None | S |
|
||||
| 2.2 | Create `FullSyncStrategy` wrapper (embeds original Syncer) | 2.1 | S |
|
||||
| 2.3 | Create `PartialSyncer` struct in `partial_sync.go` | 1.2, 2.1 | M |
|
||||
| 2.4 | Implement account processing with storage filtering | 2.3 | M |
|
||||
| 2.5 | Add `markStorageSkipped` / `isStorageSkipped` helpers | 2.3 | S |
|
||||
| 2.6 | Implement healing with skip checks | 2.5 | M |
|
||||
| 2.7 | Modify Downloader to use `SyncStrategy` interface | 2.1, 2.2 | S |
|
||||
| 2.8 | Add strategy selection based on config | 2.7 | S |
|
||||
| 2.9 | Unit tests for PartialSyncer | 2.4, 2.6 | M |
|
||||
| 2.10 | Integration test with partial filter | 2.9 | L |
|
||||
|
||||
### Phase 3: BAL Processing (State Updates)
|
||||
|
||||
| Task ID | Task | Dependencies | Effort |
|
||||
|---------|------|--------------|--------|
|
||||
| 3.1 | Add BAL key schema to `core/rawdb/schema.go` | None | S |
|
||||
| 3.2 | Create `core/rawdb/accessors_bal.go` (following existing pattern) | 3.1 | S |
|
||||
| 3.3 | Create thin `BALHistory` wrapper in `core/state/partial/history.go` | 3.2 | S |
|
||||
| 3.4 | Implement `ApplyBALAndComputeRoot` using existing BAL types + trie | Phase 2 | L |
|
||||
| 3.5 | Implement `applyStorageChanges` for tracked contracts | 3.4 | M |
|
||||
| 3.6 | Add `ProcessBlockWithBAL` to BlockChain | 3.4, 3.3 | L |
|
||||
| 3.7 | Implement reorg handling with BAL history | 3.3, 3.6 | L |
|
||||
| 3.8 | Engine API integration for BAL delivery | 3.6 | M |
|
||||
| 3.9 | BAL processing tests | 3.6, 3.7 | L |
|
||||
|
||||
### Phase 4: RPC Modifications (API Layer)
|
||||
|
||||
| Task ID | Task | Dependencies | Effort |
|
||||
|---------|------|--------------|--------|
|
||||
| 4.1 | Add `PartialStateError` and error codes | None | S |
|
||||
| 4.2 | Add `PartialStateEnabled`, `IsContractTracked` to Backend | 1.2 | S |
|
||||
| 4.3 | Modify `GetStorageAt` for partial state | 4.1, 4.2 | S |
|
||||
| 4.4 | Modify `GetCode` for partial state | 4.1, 4.2 | S |
|
||||
| 4.5 | Modify `GetProof` (account ok, storage filtered) | 4.1, 4.2 | M |
|
||||
| 4.6 | Modify `Call` / `EstimateGas` with pre-check | 4.1, 4.2 | M |
|
||||
| 4.7 | RPC behavior tests | 4.3-4.6 | M |
|
||||
|
||||
### Phase 5: Integration & Testing
|
||||
|
||||
| Task ID | Task | Dependencies | Effort |
|
||||
|---------|------|--------------|--------|
|
||||
| 5.1 | End-to-end partial sync test | Phase 2, Phase 3 | L |
|
||||
| 5.2 | Verify storage size meets estimates | 5.1 | M |
|
||||
| 5.3 | Reorg recovery test | Phase 3 | M |
|
||||
| 5.4 | RPC integration test | Phase 4, 5.1 | M |
|
||||
| 5.5 | Documentation updates | All | M |
|
||||
|
||||
### Effort Legend
|
||||
|
||||
- **S** = Small (few hours)
|
||||
- **M** = Medium (1-2 days)
|
||||
- **L** = Large (3-5 days)
|
||||
|
||||
---
|
||||
|
||||
## Critical Path
|
||||
|
||||
The critical path for minimum viable partial statefulness:
|
||||
|
||||
1. **Phase 1**: Configuration infrastructure
|
||||
2. **Phase 2**: Selective snap sync via strategy pattern (accounts + filtered storage)
|
||||
3. **Phase 3**: BAL processing (state updates without re-execution, using existing BAL types)
|
||||
4. **Phase 4**: RPC modifications (proper error handling)
|
||||
5. **Phase 5**: End-to-end test
|
||||
|
||||
This enables a working partial stateful node. Compression and full reorg handling can be added incrementally.
|
||||
|
||||
## Key Design Decisions Summary
|
||||
|
||||
| Decision | Approach | Rationale |
|
||||
|----------|----------|-----------|
|
||||
| Snap sync | Strategy pattern with separate `PartialSyncer` | Keep original `Syncer` untouched |
|
||||
| BAL types | Use existing `core/types/bal/` | Already implemented in geth master |
|
||||
| Filter interface | `ContractFilter` interface | Flexible, testable |
|
||||
| Skip tracking | DB markers + in-memory map | Persist across restarts |
|
||||
| RPC errors | Custom error codes | Clear user feedback |
|
||||
|
||||
---
|
||||
|
||||
## Reuse vs. New Code Summary
|
||||
|
||||
### REUSING (Do Not Reimplement)
|
||||
|
||||
| Component | Existing Location | How We Use It |
|
||||
|-----------|-------------------|---------------|
|
||||
| **BAL Types** | `core/types/bal/` | Import directly |
|
||||
| **Compression** | `triedb/pathdb/nodes.go` | `encodeNodeCompressed()`, `encodeNodeHistory()` |
|
||||
| **Delta Encoding** | `trie/node.go` | `NodeDifference()` |
|
||||
| **Checkpoint Mechanism** | `triedb/pathdb/config.go` | `FullValueCheckpoint` config |
|
||||
| **Diff Layers** | `triedb/pathdb/difflayer.go` | `nodeSetWithOrigin`, `StateSetWithOrigin` |
|
||||
| **History Key Patterns** | `core/rawdb/schema.go` | Follow `StateHistoryAccountBlockPrefix` pattern |
|
||||
| **History Accessors** | `core/rawdb/accessors_history.go` | Follow Read/Write/Delete triplet pattern |
|
||||
| **Safe Deletion** | `core/rawdb/database.go` | `SafeDeleteRange()` for pruning |
|
||||
| **Filter Patterns** | `eth/filters/filter.go` | Reference for contract filtering |
|
||||
| **Trie Interface** | `trie/trie.go` | Standard trie operations |
|
||||
|
||||
### CREATING NEW
|
||||
|
||||
| Component | New Location | Purpose |
|
||||
|-----------|--------------|---------|
|
||||
| `SyncStrategy` interface | `eth/protocols/snap/strategy.go` | Abstract sync implementations |
|
||||
| `PartialSyncer` | `eth/protocols/snap/partial_sync.go` | Filtered storage sync |
|
||||
| `ContractFilter` | `core/state/partial/filter.go` | Contract tracking interface |
|
||||
| `PartialState` | `core/state/partial/state.go` | BAL application + root computation |
|
||||
| BAL key schema | `core/rawdb/schema.go` | Add `balHistoryPrefix` |
|
||||
| BAL accessors | `core/rawdb/accessors_bal.go` | Read/Write/Delete following pattern |
|
||||
| `BALHistory` wrapper | `core/state/partial/history.go` | Thin layer over rawdb |
|
||||
| `ProcessBlockWithBAL` | `core/blockchain_partial.go` | Block processing entry point |
|
||||
| RPC error codes | `internal/ethapi/` | Partial state errors |
|
||||
| Config | `eth/ethconfig/config.go` | `PartialStateConfig` |
|
||||
| CLI flags | `cmd/utils/flags.go` | Partial state flags |
|
||||
760
docs/partial-state/PHASE2_PLAN.md
Normal file
760
docs/partial-state/PHASE2_PLAN.md
Normal file
|
|
@ -0,0 +1,760 @@
|
|||
# Phase 2: Snap Sync Modifications for Partial Statefulness
|
||||
|
||||
## Pre-Execution Tasks
|
||||
|
||||
Before implementing Phase 2, complete these preparatory tasks:
|
||||
|
||||
### Task 0.1: Commit Phase 1 Changes
|
||||
Commit all existing Phase 1 work (configuration, filters, BAL infrastructure):
|
||||
```bash
|
||||
git add cmd/geth/chaincmd.go cmd/geth/main.go cmd/utils/flags.go \
|
||||
core/rawdb/schema.go core/rawdb/accessors_bal.go \
|
||||
eth/ethconfig/config.go eth/ethconfig/gen_config.go \
|
||||
core/state/partial/
|
||||
git commit -m "eth: add partial statefulness foundation (Phase 1)
|
||||
|
||||
Implements EIP-7928 BAL-based partial statefulness infrastructure:
|
||||
|
||||
- Add PartialStateConfig to eth/ethconfig with CLI flags
|
||||
- Add ContractFilter interface in core/state/partial/
|
||||
- Add BAL history database accessors in core/rawdb/
|
||||
- Add PartialState and BALHistory managers
|
||||
|
||||
This enables nodes to track only configured contracts' storage
|
||||
while maintaining full account trie integrity."
|
||||
```
|
||||
|
||||
### Task 0.2: Save Plan Documentation
|
||||
Create a reference document in the repo (not to be committed):
|
||||
```bash
|
||||
mkdir -p docs/partial-state
|
||||
# Copy plan content to docs/partial-state/PHASE2_PLAN.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This plan modifies go-ethereum's snap sync to support **partial statefulness**: downloading ALL accounts but only storage/bytecode for **configured contracts**. This enables nodes to operate with ~30-40GB instead of ~1TB+ while maintaining full account trie integrity.
|
||||
|
||||
---
|
||||
|
||||
## Snap Sync Protocol Overview
|
||||
|
||||
Based on comprehensive analysis of 10 different aspects of the snap sync implementation:
|
||||
|
||||
### Current Architecture
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Syncer.Sync() │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ PHASE 1: Snap Download │ │
|
||||
│ │ 1. assignAccountTasks() → Download account ranges │ │
|
||||
│ │ 2. processAccountResponse() → Analyze each account: │ │
|
||||
│ │ • CodeHash != Empty → Add to codeTasks │ │
|
||||
│ │ • Root != Empty → Add to stateTasks │ │
|
||||
│ │ 3. assignBytecodeTasks() → Download bytecodes │ │
|
||||
│ │ 4. assignStorageTasks() → Download storage slots │ │
|
||||
│ └──────────────────────────────────────────────────────────┘ │
|
||||
│ ┌──────────────────────────────────────────────────────────┐ │
|
||||
│ │ PHASE 2: Healing │ │
|
||||
│ │ • Fill gaps in trie structure │ │
|
||||
│ │ • Download missing intermediate nodes │ │
|
||||
│ └──────────────────────────────────────────────────────────┘ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### Key Decision Points for Filtering
|
||||
|
||||
| Location | Function | Decision |
|
||||
| ------------------- | -------------------------- | -------------------------------------------------------- |
|
||||
| `sync.go:1908-1928` | `processAccountResponse()` | Checks `CodeHash != EmptyCodeHash` → adds to `codeTasks` |
|
||||
| `sync.go:1930-1969` | `processAccountResponse()` | Checks `Root != EmptyRootHash` → adds to `stateTasks` |
|
||||
| `sync.go:1117-1215` | `assignBytecodeTasks()` | Iterates `codeTasks` map |
|
||||
| `sync.go:1220-1373` | `assignStorageTasks()` | Iterates `stateTasks` map |
|
||||
|
||||
### Key Data Structures
|
||||
|
||||
```go
|
||||
type accountTask struct {
|
||||
needCode []bool // Which accounts need bytecode
|
||||
needState []bool // Which accounts need storage
|
||||
needHeal []bool // Which accounts need healing
|
||||
codeTasks map[common.Hash]struct{} // Pending bytecode hashes
|
||||
stateTasks map[common.Hash]common.Hash // Account hash → storage root
|
||||
stateCompleted map[common.Hash]struct{} // Completed storage syncs
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Design: Minimal-Invasion Approach
|
||||
|
||||
Instead of creating a separate `PartialSyncer`, we'll add **filter checks at decision points** within the existing Syncer. This is less invasive and easier to maintain.
|
||||
|
||||
### Changes Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ eth/protocols/snap/sync.go │
|
||||
│ • Add filter field to Syncer struct │
|
||||
│ • Modify processAccountResponse() to check filter │
|
||||
│ • Add skip markers for intentionally skipped storage │
|
||||
│ • Modify healing to skip intentionally-skipped accounts │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ eth/protocols/snap/sync_partial.go (NEW) │
|
||||
│ • PartialSyncConfig struct │
|
||||
│ • Skip marker database functions │
|
||||
│ • Helper functions for filter integration │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ eth/downloader/downloader.go │
|
||||
│ • Pass PartialStateConfig to snap.Syncer │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Detailed Implementation Plan
|
||||
|
||||
### Task 2.1: Add Filter to Syncer Struct
|
||||
|
||||
**File:** `eth/protocols/snap/sync.go`
|
||||
|
||||
Add filter field to Syncer:
|
||||
```go
|
||||
type Syncer struct {
|
||||
// ... existing fields ...
|
||||
|
||||
// Partial state filter (nil = sync everything)
|
||||
filter partial.ContractFilter
|
||||
}
|
||||
```
|
||||
|
||||
Modify `NewSyncer()`:
|
||||
```go
|
||||
func NewSyncer(db ethdb.KeyValueStore, scheme string, filter partial.ContractFilter) *Syncer {
|
||||
return &Syncer{
|
||||
db: db,
|
||||
scheme: scheme,
|
||||
filter: filter, // May be nil for full sync
|
||||
// ... rest unchanged
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~10 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.2: Create sync_partial.go Helper File
|
||||
|
||||
**File:** `eth/protocols/snap/sync_partial.go` (NEW)
|
||||
|
||||
```go
|
||||
package snap
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/state/partial"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
)
|
||||
|
||||
// Database key prefix for tracking intentionally skipped storage
|
||||
var skippedStoragePrefix = []byte("SnapSkipped")
|
||||
|
||||
// skippedStorageKey returns the database key for a skipped storage marker
|
||||
func skippedStorageKey(accountHash common.Hash) []byte {
|
||||
return append(skippedStoragePrefix, accountHash.Bytes()...)
|
||||
}
|
||||
|
||||
// markStorageSkipped records that storage was intentionally skipped for an account
|
||||
func markStorageSkipped(db ethdb.KeyValueWriter, accountHash common.Hash, storageRoot common.Hash) {
|
||||
db.Put(skippedStorageKey(accountHash), storageRoot.Bytes())
|
||||
}
|
||||
|
||||
// isStorageSkipped checks if storage was intentionally skipped for an account
|
||||
func isStorageSkipped(db ethdb.KeyValueReader, accountHash common.Hash) bool {
|
||||
has, _ := db.Has(skippedStorageKey(accountHash))
|
||||
return has
|
||||
}
|
||||
|
||||
// deleteStorageSkipped removes the skip marker (used during cleanup)
|
||||
func deleteStorageSkipped(db ethdb.KeyValueWriter, accountHash common.Hash) {
|
||||
db.Delete(skippedStorageKey(accountHash))
|
||||
}
|
||||
|
||||
// shouldSyncStorage returns true if storage should be synced for this address
|
||||
func (s *Syncer) shouldSyncStorage(addr common.Address) bool {
|
||||
if s.filter == nil {
|
||||
return true // No filter = sync everything
|
||||
}
|
||||
return s.filter.ShouldSyncStorage(addr)
|
||||
}
|
||||
|
||||
// shouldSyncCode returns true if bytecode should be synced for this address
|
||||
func (s *Syncer) shouldSyncCode(addr common.Address) bool {
|
||||
if s.filter == nil {
|
||||
return true // No filter = sync everything
|
||||
}
|
||||
return s.filter.ShouldSyncCode(addr)
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~50 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.3: Modify processAccountResponse() for Filtering
|
||||
|
||||
**File:** `eth/protocols/snap/sync.go`
|
||||
|
||||
**Current code (lines 1908-1969):**
|
||||
```go
|
||||
// Check if the account is a contract with an unknown code
|
||||
if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) {
|
||||
if !rawdb.HasCodeWithPrefix(s.db, common.BytesToHash(account.CodeHash)) {
|
||||
res.task.codeTasks[common.BytesToHash(account.CodeHash)] = struct{}{}
|
||||
res.task.needCode[i] = true
|
||||
res.task.pend++
|
||||
}
|
||||
}
|
||||
// Check if the account is a contract with an unknown storage trie
|
||||
if account.Root != types.EmptyRootHash {
|
||||
// ... adds to stateTasks
|
||||
}
|
||||
```
|
||||
|
||||
**Modified code:**
|
||||
```go
|
||||
// Derive address from account hash for filter check
|
||||
// Note: We have the hash, need to track address mapping
|
||||
addr := s.hashToAddress(res.hashes[i]) // New helper needed
|
||||
|
||||
// Check if the account is a contract with an unknown code
|
||||
if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) {
|
||||
if !rawdb.HasCodeWithPrefix(s.db, common.BytesToHash(account.CodeHash)) {
|
||||
// NEW: Check filter before adding to codeTasks
|
||||
if s.shouldSyncCode(addr) {
|
||||
res.task.codeTasks[common.BytesToHash(account.CodeHash)] = struct{}{}
|
||||
res.task.needCode[i] = true
|
||||
res.task.pend++
|
||||
}
|
||||
// If filtered out, bytecode just won't be fetched
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the account is a contract with an unknown storage trie
|
||||
if account.Root != types.EmptyRootHash {
|
||||
// NEW: Check filter before adding to stateTasks
|
||||
if s.shouldSyncStorage(addr) {
|
||||
// ... existing logic to add to stateTasks
|
||||
} else {
|
||||
// Mark as intentionally skipped for healing phase
|
||||
markStorageSkipped(s.db, res.hashes[i], account.Root)
|
||||
res.task.stateCompleted[res.hashes[i]] = struct{}{}
|
||||
// Don't increment pend - we're not waiting for this storage
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Challenge:** We have account hashes but need addresses for filter checks.
|
||||
|
||||
**Solution:** The filter operates on addresses, but snap sync uses hashes. Two options:
|
||||
1. Store hash→address mapping during sync (memory overhead)
|
||||
2. Modify filter to work with hashes (requires pre-computing hashes of configured addresses)
|
||||
|
||||
**Recommended: Option 2** - Pre-compute hashes in filter:
|
||||
```go
|
||||
type ConfiguredFilter struct {
|
||||
contracts map[common.Address]struct{}
|
||||
contractHashes map[common.Hash]struct{} // Pre-computed: keccak256(address)
|
||||
}
|
||||
|
||||
func (f *ConfiguredFilter) ShouldSyncStorageByHash(hash common.Hash) bool {
|
||||
_, ok := f.contractHashes[hash]
|
||||
return ok
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~40 lines in sync.go, ~20 lines in filter.go
|
||||
|
||||
---
|
||||
|
||||
### Task 2.4: Modify Healing to Skip Storage for Non-Tracked Contracts
|
||||
|
||||
**Important Clarification:** We **NEVER skip accounts** - ALL accounts are always synced (this is the core value proposition). We only skip **storage and bytecode** for contracts not in the configured filter.
|
||||
|
||||
**File:** `eth/protocols/snap/sync.go`
|
||||
|
||||
In `onHealState()` callback (lines 3071-3092), add check for **storage leaves only**:
|
||||
```go
|
||||
func (s *Syncer) onHealState(paths [][]byte, value []byte) error {
|
||||
if len(paths) == 1 {
|
||||
// Account trie leaf - ALWAYS process (never skip accounts)
|
||||
var account types.StateAccount
|
||||
if err := rlp.DecodeBytes(value, &account); err != nil {
|
||||
return nil
|
||||
}
|
||||
blob := types.SlimAccountRLP(account)
|
||||
rawdb.WriteAccountSnapshot(s.stateWriter, common.BytesToHash(paths[0]), blob)
|
||||
s.accountHealed += 1
|
||||
// ... rest unchanged
|
||||
}
|
||||
if len(paths) == 2 {
|
||||
// Storage trie leaf
|
||||
accountHash := common.BytesToHash(paths[0])
|
||||
|
||||
// NEW: Skip STORAGE healing for non-tracked contracts
|
||||
// (accounts themselves are always synced/healed)
|
||||
if isStorageSkipped(s.db, accountHash) {
|
||||
return nil // Don't heal storage we intentionally skipped
|
||||
}
|
||||
|
||||
// ... existing storage handling
|
||||
rawdb.WriteStorageSnapshot(s.stateWriter, accountHash, ...)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
```
|
||||
|
||||
Also modify healing task creation to avoid requesting storage trie nodes for non-tracked contracts.
|
||||
|
||||
**Key principle:** Account healing always proceeds. Only storage trie node requests are filtered.
|
||||
|
||||
**Estimated changes:** ~30 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.5: Update Downloader to Pass Filter
|
||||
|
||||
**File:** `eth/downloader/downloader.go`
|
||||
|
||||
Modify `New()` to accept and pass filter:
|
||||
```go
|
||||
func New(stateDb ethdb.Database, mode ethconfig.SyncMode, ...,
|
||||
partialConfig *ethconfig.PartialStateConfig) *Downloader {
|
||||
|
||||
var filter partial.ContractFilter
|
||||
if partialConfig != nil && partialConfig.Enabled {
|
||||
filter = partial.NewConfiguredFilter(partialConfig.Contracts)
|
||||
}
|
||||
|
||||
dl := &Downloader{
|
||||
// ... existing fields
|
||||
SnapSyncer: snap.NewSyncer(stateDb, chain.TrieDB().Scheme(), filter),
|
||||
}
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**File:** `eth/handler.go`
|
||||
|
||||
Pass config through handler:
|
||||
```go
|
||||
h.downloader = downloader.New(config.Database, config.Sync, h.eventMux,
|
||||
h.chain, h.removePeer, h.enableSyncedFeatures,
|
||||
&config.Eth.PartialState)
|
||||
```
|
||||
|
||||
**Estimated changes:** ~20 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.6: Add Hash-Based Filter Methods
|
||||
|
||||
**File:** `core/state/partial/filter.go`
|
||||
|
||||
Extend ConfiguredFilter:
|
||||
```go
|
||||
type ConfiguredFilter struct {
|
||||
contracts map[common.Address]struct{}
|
||||
contractHashes map[common.Hash]struct{} // NEW: Pre-computed hashes
|
||||
}
|
||||
|
||||
func NewConfiguredFilter(addresses []common.Address) *ConfiguredFilter {
|
||||
m := make(map[common.Address]struct{}, len(addresses))
|
||||
h := make(map[common.Hash]struct{}, len(addresses))
|
||||
for _, addr := range addresses {
|
||||
m[addr] = struct{}{}
|
||||
h[crypto.Keccak256Hash(addr.Bytes())] = struct{}{} // Pre-compute hash
|
||||
}
|
||||
return &ConfiguredFilter{contracts: m, contractHashes: h}
|
||||
}
|
||||
|
||||
// NEW: Hash-based filter for snap sync (which works with hashes, not addresses)
|
||||
func (f *ConfiguredFilter) ShouldSyncStorageByHash(hash common.Hash) bool {
|
||||
_, ok := f.contractHashes[hash]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (f *ConfiguredFilter) ShouldSyncCodeByHash(hash common.Hash) bool {
|
||||
_, ok := f.contractHashes[hash]
|
||||
return ok
|
||||
}
|
||||
```
|
||||
|
||||
Update ContractFilter interface:
|
||||
```go
|
||||
type ContractFilter interface {
|
||||
ShouldSyncStorage(address common.Address) bool
|
||||
ShouldSyncCode(address common.Address) bool
|
||||
IsTracked(address common.Address) bool
|
||||
|
||||
// Hash-based methods for snap sync
|
||||
ShouldSyncStorageByHash(hash common.Hash) bool
|
||||
ShouldSyncCodeByHash(hash common.Hash) bool
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~30 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.7: Persist Skip Markers for Resumption
|
||||
|
||||
**File:** `eth/protocols/snap/sync.go`
|
||||
|
||||
In `saveSyncStatus()`, ensure skip markers are preserved (they're already in DB, just verify):
|
||||
```go
|
||||
func (s *Syncer) saveSyncStatus() {
|
||||
// ... existing serialization
|
||||
|
||||
// Skip markers are already in DB (written during processAccountResponse)
|
||||
// They persist across restarts automatically
|
||||
}
|
||||
```
|
||||
|
||||
In `loadSyncStatus()`, log skipped storage count for visibility:
|
||||
```go
|
||||
func (s *Syncer) loadSyncStatus() {
|
||||
// ... existing deserialization
|
||||
|
||||
if s.filter != nil {
|
||||
log.Info("Partial state sync active",
|
||||
"trackedContracts", len(s.filter.Contracts()))
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~10 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.8: Add Metrics for Partial Sync
|
||||
|
||||
**File:** `eth/protocols/snap/sync.go`
|
||||
|
||||
Add counters:
|
||||
```go
|
||||
var (
|
||||
storageSkippedGauge = metrics.NewRegisteredGauge("snap/sync/storage/skipped", nil)
|
||||
bytecodeSkippedGauge = metrics.NewRegisteredGauge("snap/sync/bytecode/skipped", nil)
|
||||
)
|
||||
```
|
||||
|
||||
Increment in processAccountResponse:
|
||||
```go
|
||||
if !s.shouldSyncStorage(addr) {
|
||||
storageSkippedGauge.Inc(1)
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~15 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.9: Unit Tests
|
||||
|
||||
**File:** `eth/protocols/snap/sync_partial_test.go` (NEW)
|
||||
|
||||
```go
|
||||
package snap
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/state/partial"
|
||||
)
|
||||
|
||||
func TestPartialSyncFilterStorage(t *testing.T) {
|
||||
// Create filter with specific contracts
|
||||
tracked := []common.Address{
|
||||
common.HexToAddress("0x1234..."),
|
||||
}
|
||||
filter := partial.NewConfiguredFilter(tracked)
|
||||
|
||||
// Verify tracked contracts pass filter
|
||||
if !filter.ShouldSyncStorage(tracked[0]) {
|
||||
t.Error("Tracked contract should pass filter")
|
||||
}
|
||||
|
||||
// Verify untracked contracts are filtered
|
||||
untracked := common.HexToAddress("0xABCD...")
|
||||
if filter.ShouldSyncStorage(untracked) {
|
||||
t.Error("Untracked contract should be filtered")
|
||||
}
|
||||
|
||||
// Verify hash-based filter works
|
||||
trackedHash := crypto.Keccak256Hash(tracked[0].Bytes())
|
||||
if !filter.ShouldSyncStorageByHash(trackedHash) {
|
||||
t.Error("Tracked contract hash should pass filter")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkipMarkerPersistence(t *testing.T) {
|
||||
db := rawdb.NewMemoryDatabase()
|
||||
accountHash := common.HexToHash("0x1234...")
|
||||
storageRoot := common.HexToHash("0xABCD...")
|
||||
|
||||
// Mark as skipped
|
||||
markStorageSkipped(db, accountHash, storageRoot)
|
||||
|
||||
// Verify marker persists
|
||||
if !isStorageSkipped(db, accountHash) {
|
||||
t.Error("Skip marker should persist")
|
||||
}
|
||||
|
||||
// Delete and verify
|
||||
deleteStorageSkipped(db, accountHash)
|
||||
if isStorageSkipped(db, accountHash) {
|
||||
t.Error("Skip marker should be deleted")
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~100 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 2.10: Integration Test
|
||||
|
||||
**File:** `eth/protocols/snap/sync_partial_integration_test.go` (NEW)
|
||||
|
||||
Create end-to-end test that:
|
||||
1. Sets up a mock state with multiple contracts
|
||||
2. Configures partial sync with subset of contracts
|
||||
3. Runs sync
|
||||
4. Verifies:
|
||||
- All accounts synced
|
||||
- Only configured contracts have storage
|
||||
- Skip markers present for non-configured contracts
|
||||
- Healing doesn't try to heal skipped storage
|
||||
|
||||
**Estimated changes:** ~200 lines
|
||||
|
||||
---
|
||||
|
||||
## Local Testing Strategy
|
||||
|
||||
### 1. Unit Test Execution
|
||||
```bash
|
||||
cd eth/protocols/snap
|
||||
go test -v -run TestPartialSync
|
||||
go test -v -run TestSkipMarker
|
||||
```
|
||||
|
||||
### 2. Build Verification
|
||||
```bash
|
||||
go build ./...
|
||||
go build ./cmd/geth
|
||||
```
|
||||
|
||||
### 3. Simulated Network Test
|
||||
|
||||
Create a test script that:
|
||||
```bash
|
||||
# Terminal 1: Start full node (serves as peer)
|
||||
./geth --datadir /tmp/full-node --syncmode snap --port 30303
|
||||
|
||||
# Terminal 2: Start partial node
|
||||
./geth --datadir /tmp/partial-node --syncmode snap --port 30304 \
|
||||
--partial-state \
|
||||
--partial-state.contracts 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2 \
|
||||
--bootnodes "enode://..."
|
||||
```
|
||||
|
||||
### 4. Verification Checks
|
||||
|
||||
After sync completes:
|
||||
```bash
|
||||
# Check database size (should be significantly smaller)
|
||||
du -sh /tmp/partial-node/geth/chaindata
|
||||
|
||||
# Query RPC to verify:
|
||||
# - Account balance works for any address
|
||||
curl -X POST -H "Content-Type: application/json" \
|
||||
--data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["0x...", "latest"],"id":1}' \
|
||||
http://localhost:8545
|
||||
|
||||
# - Storage works for tracked contracts
|
||||
curl -X POST -H "Content-Type: application/json" \
|
||||
--data '{"jsonrpc":"2.0","method":"eth_getStorageAt","params":["0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2", "0x0", "latest"],"id":1}' \
|
||||
http://localhost:8545
|
||||
|
||||
# - Storage fails for untracked contracts (once RPC phase implemented)
|
||||
```
|
||||
|
||||
### 5. Devnet Testing
|
||||
|
||||
For full integration testing:
|
||||
1. Use a local devnet with known state
|
||||
2. Configure partial sync with specific test contracts
|
||||
3. Verify sync completion and state correctness
|
||||
4. Test reorg handling with BAL history
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify Summary
|
||||
|
||||
| File | Changes | Lines |
|
||||
| ----------------------------------------------------- | -------------------------------------------------------- | ----- |
|
||||
| `eth/protocols/snap/sync.go` | Add filter field, modify processAccountResponse, healing | ~80 |
|
||||
| `eth/protocols/snap/sync_partial.go` | NEW: Skip markers, helpers | ~50 |
|
||||
| `core/state/partial/filter.go` | Add hash-based filter methods | ~30 |
|
||||
| `eth/downloader/downloader.go` | Pass filter to Syncer | ~15 |
|
||||
| `eth/handler.go` | Pass config through | ~5 |
|
||||
| `eth/protocols/snap/sync_partial_test.go` | NEW: Unit tests | ~100 |
|
||||
| `eth/protocols/snap/sync_partial_integration_test.go` | NEW: Integration tests | ~200 |
|
||||
|
||||
**Total estimated changes:** ~480 lines
|
||||
|
||||
---
|
||||
|
||||
## Task Summary
|
||||
|
||||
| Task ID | Description | Dependencies | Effort |
|
||||
| ------- | -------------------------------- | ------------- | ------ |
|
||||
| 2.1 | Add filter to Syncer struct | None | S |
|
||||
| 2.2 | Create sync_partial.go helpers | 2.1 | S |
|
||||
| 2.3 | Modify processAccountResponse | 2.1, 2.2, 2.6 | M |
|
||||
| 2.4 | Modify healing to skip filtered | 2.2 | S |
|
||||
| 2.5 | Update Downloader to pass filter | 2.1 | S |
|
||||
| 2.6 | Add hash-based filter methods | None | S |
|
||||
| 2.7 | Persist skip markers | 2.2 | S |
|
||||
| 2.8 | Add metrics | 2.3 | S |
|
||||
| 2.9 | Unit tests | 2.1-2.8 | M |
|
||||
| 2.10 | Integration test | 2.9 | L |
|
||||
|
||||
**Effort:** S = Small (few hours), M = Medium (1-2 days), L = Large (3-5 days)
|
||||
|
||||
---
|
||||
|
||||
## Critical Design Decisions
|
||||
|
||||
1. **Minimal Invasion**: Modify existing Syncer rather than creating parallel implementation
|
||||
2. **Hash-Based Filtering**: Pre-compute address hashes for efficient snap sync filtering
|
||||
3. **Skip Markers in DB**: Persist skip decisions for healing phase and resumption
|
||||
4. **Healing Awareness**: Healing phase must respect skip markers to avoid errors
|
||||
|
||||
---
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
After implementation:
|
||||
- [ ] **ALL accounts synced** (complete account trie - this is critical)
|
||||
- [ ] **Account trie fully intact** - eth_getProof works for ANY account
|
||||
- [ ] Only configured contracts have storage synced
|
||||
- [ ] Only configured contracts have bytecode synced
|
||||
- [ ] Skip markers present for non-tracked contract storage
|
||||
- [ ] Healing completes successfully (doesn't try to heal skipped storage)
|
||||
- [ ] Sync can resume after restart (skip markers persist)
|
||||
- [ ] Database size significantly reduced (~30-40GB vs ~1TB+)
|
||||
- [ ] eth_getBalance works for ANY account (not just tracked)
|
||||
- [ ] eth_getTransactionCount works for ANY account
|
||||
- [ ] eth_getStorageAt works for tracked contracts
|
||||
- [ ] Metrics show skipped storage/bytecode counts
|
||||
|
||||
**Critical Invariant:** The account trie must be 100% complete. Only storage tries for non-tracked contracts are skipped.
|
||||
|
||||
|
||||
# After implemenmtation:
|
||||
★ Insight ─────────────────────────────────────
|
||||
Phase 2 Implementation Complete:
|
||||
The key architectural change is that snap sync now supports partial statefulness through a filter mechanism. The filter is passed through
|
||||
the entire stack:
|
||||
1. ethconfig.PartialStateConfig → CLI configuration
|
||||
2. partial.ConfiguredFilter → Pre-computes keccak256 hashes for efficient lookup
|
||||
3. snap.Syncer.filter → Makes filtering decisions during sync
|
||||
4. markStorageSkipped() / isStorageSkipped() → Persists skip decisions for healing phase
|
||||
|
||||
This allows a node to sync ~30-40GB (all accounts) instead of ~1TB+ (all accounts + all storage).
|
||||
─────────────────────────────────────────────────
|
||||
|
||||
⏺ Bash(git diff --stat HEAD 2>&1 | tail -20)
|
||||
⎿ core/state/partial/filter.go | 46 +++++++++++++++++++++++++--
|
||||
eth/backend.go | 10 ++++++
|
||||
eth/downloader/downloader.go | 5 +--
|
||||
… +5 lines (ctrl+o to expand)
|
||||
|
||||
⏺ Bash(git status --short 2>&1)
|
||||
⎿ M core/state/partial/filter.go
|
||||
M eth/backend.go
|
||||
M eth/downloader/downloader.go
|
||||
… +9 lines (ctrl+o to expand)
|
||||
|
||||
⏺ Here's a summary of the Phase 2 Snap Sync Modifications that have been implemented:
|
||||
|
||||
Phase 2 Implementation Summary
|
||||
|
||||
Tasks Completed
|
||||
┌──────┬───────────────────────────────────────────────┬───────────────────────────────────┐
|
||||
│ Task │ Description │ Status │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.1 │ Add filter to Syncer struct │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.2 │ Create sync_partial.go helpers │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.3 │ Modify processAccountResponse() for filtering │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.4 │ Modify healing to skip storage │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.5 │ Update Downloader to pass filter │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.6 │ Add hash-based filter methods │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.7 │ Persist skip markers │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.8 │ Add metrics │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.9 │ Unit tests │ ✓ │
|
||||
├──────┼───────────────────────────────────────────────┼───────────────────────────────────┤
|
||||
│ 2.10 │ Integration test │ Deferred (requires network setup) │
|
||||
└──────┴───────────────────────────────────────────────┴───────────────────────────────────┘
|
||||
Files Modified (7 files, +112 lines)
|
||||
|
||||
- core/state/partial/filter.go - Added hash-based filter methods
|
||||
- eth/backend.go - Create and pass partial filter
|
||||
- eth/downloader/downloader.go - Accept filter parameter
|
||||
- eth/handler.go - Add PartialFilter to handlerConfig
|
||||
- eth/protocols/snap/sync.go - Filter storage/bytecode in processAccountResponse and healing
|
||||
|
||||
Files Created (4 files)
|
||||
|
||||
- eth/protocols/snap/sync_partial.go - Skip marker DB functions and filter helpers
|
||||
- eth/protocols/snap/sync_partial_test.go - Unit tests for partial sync
|
||||
- core/state/partial/filter_test.go - Unit tests for filter
|
||||
- docs/partial-state/PHASE2_PLAN.md - Plan documentation
|
||||
|
||||
Key Design Decisions
|
||||
|
||||
1. Minimal invasion: Modified existing Syncer rather than creating parallel implementation
|
||||
2. Hash-based filtering: Pre-compute keccak256 hashes for O(1) lookup during sync
|
||||
3. Skip markers: Persist decisions in DB so healing phase knows what was intentionally skipped
|
||||
4. All accounts synced: Only storage/bytecode are filtered, account trie is always complete
|
||||
|
||||
Testing
|
||||
|
||||
All 8 new tests pass:
|
||||
- TestPartialSyncFilterStorage
|
||||
- TestAllowAllFilter
|
||||
- TestSkipMarkerPersistence
|
||||
- TestSyncerFilterMethods
|
||||
- TestConfiguredFilterContracts
|
||||
- TestConfiguredFilterBasic
|
||||
- TestConfiguredFilterHashConsistency
|
||||
- TestAllowAllFilterInterface
|
||||
445
docs/partial-state/PHASE3_PLAN.md
Normal file
445
docs/partial-state/PHASE3_PLAN.md
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
# Phase 3: BAL Processing & State Updates for Partial Statefulness
|
||||
|
||||
## Overview
|
||||
|
||||
**Goal**: Enable partial state nodes to process blocks using Block Access Lists (BALs) instead of re-executing transactions. This allows state updates without needing full contract storage.
|
||||
|
||||
**Key principle**: BALs (per EIP-7928) provide state diffs that allow computing the new state root by applying changes directly to the trie, without transaction execution.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Phase 1 (Configuration & Infrastructure): ✓ Complete
|
||||
- Phase 2 (Snap Sync Modifications): ✓ Complete
|
||||
- EIP-7928 BAL types already exist in `core/types/bal/`
|
||||
|
||||
---
|
||||
|
||||
## Design Overview
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ Block Processing Flow │
|
||||
│ │
|
||||
│ Full Node: Block → Execute TXs → Compute State Root │
|
||||
│ │
|
||||
│ Partial Node: Block + BAL → Apply BAL Diffs → Verify Root │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
|
||||
┌─────────────────────────────────────────────────────────────────┐
|
||||
│ BAL Application Flow │
|
||||
│ │
|
||||
│ 1. Receive block + BAL (via Engine API) │
|
||||
│ 2. Verify: keccak256(rlp(BAL)) == header.BlockAccessListHash │
|
||||
│ 3. For each AccountAccess in BAL: │
|
||||
│ a. Load account from trie │
|
||||
│ b. Apply balance/nonce changes (final values) │
|
||||
│ c. Apply storage changes (tracked contracts only) │
|
||||
│ d. Update account in trie │
|
||||
│ 4. Commit trie → Verify root matches header.stateRoot │
|
||||
│ 5. Store BAL for reorg handling │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Existing Infrastructure (ALREADY EXISTS - REUSE!)
|
||||
|
||||
Based on agent exploration, the following infrastructure **already exists and is production-ready**:
|
||||
|
||||
| Component | Location | Status |
|
||||
|-----------|----------|--------|
|
||||
| BAL Types | `core/types/bal/bal.go` | ✅ Complete - `ConstructionBlockAccessList`, `BlockAccessList` |
|
||||
| BAL Encoding | `core/types/bal/bal_encoding.go` | ✅ Complete - RLP, Hash(), Validate() |
|
||||
| DB Schema | `core/rawdb/schema.go:172` | ✅ Complete - prefix `"p"` |
|
||||
| DB Accessors | `core/rawdb/accessors_bal.go` | ✅ Complete - Read/Write/Delete/Prune |
|
||||
| BALHistory | `core/state/partial/history.go` | ✅ Complete - wrapper over rawdb |
|
||||
| PartialState | `core/state/partial/state.go` | ⚠️ Skeleton - needs `ApplyBALAndComputeRoot()` |
|
||||
| ContractFilter | `core/state/partial/filter.go` | ✅ Complete - ConfiguredFilter, AllowAllFilter |
|
||||
| Trie Interface | `trie/trie.go` | ✅ Standard trie operations |
|
||||
|
||||
**What this means:** Tasks 3.2, 3.3, 3.4 are already done! We only need to implement:
|
||||
- `ApplyBALAndComputeRoot()` in PartialState
|
||||
- `ProcessBlockWithBAL()` in BlockChain
|
||||
- Reorg handling
|
||||
- Tests
|
||||
|
||||
---
|
||||
|
||||
## Detailed Implementation Plan
|
||||
|
||||
### Task 3.1: Review/Extend Existing PartialState Struct
|
||||
|
||||
**File:** `core/state/partial/state.go` (ALREADY EXISTS!)
|
||||
|
||||
**Agent Finding:** PartialState skeleton already exists with correct structure:
|
||||
```go
|
||||
type PartialState struct {
|
||||
db ethdb.Database
|
||||
trieDB *triedb.Database
|
||||
filter ContractFilter
|
||||
history *BALHistory // Already includes history!
|
||||
stateRoot common.Hash
|
||||
}
|
||||
```
|
||||
|
||||
**Current methods (already implemented):**
|
||||
- `NewPartialState()` - Constructor ✅
|
||||
- `Filter()` - Filter access ✅
|
||||
- `Root()` / `SetRoot()` - Root management ✅
|
||||
- `History()` - BAL history access ✅
|
||||
|
||||
**Key patterns from StateDB (confirmed by agent):**
|
||||
- PartialState does NOT need `stateObjects` caching (applies BAL directly to trie)
|
||||
- PartialState does NOT need journal/revert (BAL diffs are immutable)
|
||||
- PartialState does NOT need prefetcher (not executing contracts)
|
||||
- Error handling: return errors immediately (no memoization)
|
||||
|
||||
**What needs to be added:**
|
||||
- `ApplyBALAndComputeRoot()` method (Task 3.5)
|
||||
- Optional metrics fields for monitoring
|
||||
|
||||
**Estimated changes:** ~10 lines (mostly just adding ApplyBALAndComputeRoot)
|
||||
|
||||
---
|
||||
|
||||
### Task 3.2: ✅ ALREADY EXISTS - BAL History Database Schema
|
||||
|
||||
**File:** `core/rawdb/schema.go` line 172
|
||||
|
||||
**Agent confirmed:** Schema already exists!
|
||||
```go
|
||||
balHistoryPrefix = []byte("p") // balHistoryPrefix + num (uint64 big endian) -> RLP(bal.BlockAccessList)
|
||||
```
|
||||
|
||||
**Key format:** `"p" + blockNumber(8 bytes, big-endian)` → RLP-encoded BlockAccessList
|
||||
|
||||
**Estimated changes:** 0 lines (already exists)
|
||||
|
||||
---
|
||||
|
||||
### Task 3.3: ✅ ALREADY EXISTS - BAL History Accessors
|
||||
|
||||
**File:** `core/rawdb/accessors_bal.go`
|
||||
|
||||
**Agent confirmed:** All accessors already implemented!
|
||||
- `ReadBALHistory(db, blockNum)` ✅
|
||||
- `WriteBALHistory(db, blockNum, accessList)` ✅
|
||||
- `DeleteBALHistory(db, blockNum)` ✅
|
||||
- `HasBALHistory(db, blockNum)` ✅
|
||||
- `PruneBALHistory(db, beforeBlock)` ✅ (with safe range iteration)
|
||||
|
||||
**Estimated changes:** 0 lines (already exists)
|
||||
|
||||
---
|
||||
|
||||
### Task 3.4: ✅ ALREADY EXISTS - BALHistory Wrapper
|
||||
|
||||
**File:** `core/state/partial/history.go`
|
||||
|
||||
**Agent confirmed:** BALHistory wrapper already implemented!
|
||||
```go
|
||||
type BALHistory struct {
|
||||
db ethdb.Database
|
||||
retention uint64
|
||||
}
|
||||
|
||||
// Methods: Store(), Get(), Delete(), Prune(), Retention()
|
||||
```
|
||||
|
||||
**Design note:** We have BOTH:
|
||||
1. BALHistory in `partial/history.go` - for explicit BAL storage/retrieval
|
||||
2. Blocks contain BALs - can also access via block
|
||||
|
||||
For reorgs, we'll use BALHistory since it's already built and tested.
|
||||
|
||||
**Estimated changes:** 0 lines (already exists)
|
||||
|
||||
---
|
||||
|
||||
### Task 3.5: Implement ApplyBALAndComputeRoot
|
||||
|
||||
**File:** `core/state/partial/state.go` (extend)
|
||||
|
||||
**Key implementation requirements (from code review and agent research):**
|
||||
|
||||
1. **BAL field names**: Use `Accesses` (not `Writes`) and `ValueAfter` (not `Value`) per `core/types/bal/bal_encoding.go`
|
||||
2. **Commit ordering**: Storage tries → update account.Root → account trie (critical for correct state root)
|
||||
3. **Account origin tracking**: Track `existed` flag to prevent incorrect EIP-161 deletion
|
||||
4. **Code handling**: Update CodeHash for ALL accounts, store code bytes only for tracked contracts
|
||||
5. **PathDB StateSet**: Must construct proper `triedb.StateSet` for `trieDB.Update()` call
|
||||
|
||||
**PathDB StateSet construction (from agent research on `core/state/statedb.go`):**
|
||||
|
||||
The `trieDB.Update()` signature is:
|
||||
```go
|
||||
func (db *Database) Update(root, parent common.Hash, block uint64, nodes *trienode.MergedNodeSet, states *StateSet) error
|
||||
```
|
||||
|
||||
The `StateSet` structure requires:
|
||||
```go
|
||||
type StateSet struct {
|
||||
Accounts map[common.Hash][]byte // Mutated accounts in 'slim RLP' encoding
|
||||
AccountsOrigin map[common.Address][]byte // Original account values (for PathDB)
|
||||
Storages map[common.Hash]map[common.Hash][]byte // Storage: accountHash → slotHash → value
|
||||
StoragesOrigin map[common.Address]map[common.Hash][]byte // Original storage values
|
||||
RawStorageKey bool // false = use hashed keys
|
||||
}
|
||||
```
|
||||
|
||||
**Key encoding requirements:**
|
||||
- Accounts: Use `types.SlimAccountRLP(account)` for encoding
|
||||
- Storage values: Use prefix-zero-trimmed RLP (`rlp.EncodeToBytes(common.TrimLeftZeroes(val[:]))`)
|
||||
- Storage keys: Must be hashed (`crypto.Keccak256Hash(rawKey[:])`)
|
||||
- Nil values indicate deletion
|
||||
|
||||
**Estimated changes:** ~250 lines (includes PathDB StateSet construction)
|
||||
|
||||
---
|
||||
|
||||
### Task 3.6: Implement ProcessBlockWithBAL
|
||||
|
||||
**File:** `core/blockchain_partial.go` (NEW)
|
||||
|
||||
**Trust Model:** Blocks via Engine API are pre-attested by the Consensus Layer. The function documents this trust model clearly in its comments, explaining why no additional attestation verification is needed (same as full nodes).
|
||||
|
||||
**Estimated changes:** ~100 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 3.7: Implement Reorg Handling
|
||||
|
||||
**File:** `core/blockchain_partial.go` (extend)
|
||||
|
||||
**DESIGN:** Reorg handling accesses blocks directly (which contain BALs), NOT a separate BALHistory. This mirrors how full nodes handle reorgs.
|
||||
|
||||
**Key differences from full node reorg:**
|
||||
- Full node: re-executes transactions on new chain
|
||||
- Partial node: applies BALs from new chain blocks
|
||||
|
||||
**Estimated changes:** ~50 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 3.8: Wire PartialState into BlockChain
|
||||
|
||||
**File:** `core/blockchain.go` (modify)
|
||||
|
||||
**Agent findings on BlockChain state patterns:**
|
||||
|
||||
**Existing state fields (lines 311-366):**
|
||||
```go
|
||||
type BlockChain struct {
|
||||
db ethdb.Database // Low-level persistent database
|
||||
snaps *snapshot.Tree // Snapshot tree for fast trie leaf access
|
||||
triedb *triedb.Database // TrieDB handler for maintaining trie nodes
|
||||
statedb *state.CachingDB // State database (reused between imports)
|
||||
// ... caches, processor, validator, etc.
|
||||
}
|
||||
```
|
||||
|
||||
**Add partialState alongside existing fields:**
|
||||
```go
|
||||
type BlockChain struct {
|
||||
// ... existing fields ...
|
||||
|
||||
// Partial state management (nil if full node)
|
||||
partialState *partial.PartialState
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~40 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 3.9: Add Unit Tests
|
||||
|
||||
**File:** `core/state/partial/state_test.go` (NEW)
|
||||
|
||||
```go
|
||||
func TestApplyBALAndComputeRoot(t *testing.T) {
|
||||
// Test that BAL application produces correct state root
|
||||
}
|
||||
|
||||
func TestApplyStorageChanges(t *testing.T) {
|
||||
// Test storage updates for tracked contracts
|
||||
}
|
||||
|
||||
func TestApplyBalanceChanges(t *testing.T) {
|
||||
// Test balance updates from BAL
|
||||
}
|
||||
|
||||
func TestFilteredStorageChanges(t *testing.T) {
|
||||
// Test that untracked contract storage is not applied
|
||||
}
|
||||
```
|
||||
|
||||
**Estimated changes:** ~100 lines
|
||||
|
||||
---
|
||||
|
||||
### Task 3.10: Integration Test
|
||||
|
||||
**File:** `core/blockchain_partial_test.go` (NEW)
|
||||
|
||||
Test end-to-end BAL processing:
|
||||
1. Create a chain with known state
|
||||
2. Generate BALs for blocks
|
||||
3. Process blocks with `ProcessBlockWithBAL`
|
||||
4. Verify state roots match
|
||||
5. Test reorg handling
|
||||
|
||||
**Estimated changes:** ~200 lines
|
||||
|
||||
---
|
||||
|
||||
## Files to Modify/Create Summary
|
||||
|
||||
| File | Status | Changes |
|
||||
|------|--------|---------|
|
||||
| `docs/partial-state/PARTIAL_STATEFULNESS_PLAN.md` | NEW | Copy master plan from `.claude/plans/` |
|
||||
| `docs/partial-state/PHASE3_PLAN.md` | NEW | Copy this Phase 3 plan |
|
||||
| `core/state/partial/state.go` | EXTEND | Add `ApplyBALAndComputeRoot()` + StateSet (~250 lines) |
|
||||
| `core/rawdb/schema.go` | ✅ EXISTS | `balHistoryPrefix` already defined |
|
||||
| `core/rawdb/accessors_bal.go` | ✅ EXISTS | All accessors already implemented |
|
||||
| `core/state/partial/history.go` | ✅ EXISTS | `BALHistory` wrapper already implemented |
|
||||
| `core/blockchain.go` | MODIFY | Add `partialState` field, initialization (~40 lines) |
|
||||
| `core/blockchain_partial.go` | NEW | `ProcessBlockWithBAL`, reorg, attestation (~150 lines) |
|
||||
| `core/state/partial/state_test.go` | NEW | Unit tests (~100 lines) |
|
||||
| `core/blockchain_partial_test.go` | NEW | Integration tests (~200 lines) |
|
||||
|
||||
**Total estimated new code:** ~710 lines
|
||||
**Infrastructure already exists:** ~300 lines (schema, accessors, history)
|
||||
|
||||
---
|
||||
|
||||
## Task Summary
|
||||
|
||||
| Task ID | Description | Dependencies | Effort | Status |
|
||||
|---------|-------------|--------------|--------|--------|
|
||||
| 1 | Save master plan + Phase 3 plan to docs/partial-state/ | None | S | TODO |
|
||||
| 3.1 | Review existing PartialState, add metrics | Phase 1 | S | Exists |
|
||||
| 3.2 | BAL history DB schema | None | - | ✅ EXISTS |
|
||||
| 3.3 | BAL history accessors | 3.2 | - | ✅ EXISTS |
|
||||
| 3.4 | BALHistory wrapper | 3.3 | - | ✅ EXISTS |
|
||||
| 3.5 | Implement `ApplyBALAndComputeRoot` with PathDB StateSet | 3.1 | L | TODO |
|
||||
| 3.6 | Implement `ProcessBlockWithBAL` with trust model docs | 3.5 | M | TODO |
|
||||
| 3.7 | Implement reorg handling (uses BALHistory) | 3.6 | M | TODO |
|
||||
| 3.8 | Wire into BlockChain | 3.6 | S | TODO |
|
||||
| 3.9 | Unit tests | 3.5, 3.7 | M | TODO |
|
||||
| 3.10 | Integration test | 3.6, 3.7 | L | TODO |
|
||||
|
||||
**Effort:** S = Small (few hours), M = Medium (1-2 days), L = Large (3-5 days)
|
||||
|
||||
**Good news:** Tasks 3.2, 3.3, 3.4 are already implemented! Only need to implement 3.5-3.10.
|
||||
|
||||
---
|
||||
|
||||
## Dependency Graph
|
||||
|
||||
```
|
||||
Task 1 (Save master plan + Phase 3 plan)
|
||||
↓
|
||||
3.1 (Review existing PartialState) ─── 3.2/3.3/3.4 ✅ ALREADY EXIST
|
||||
↓
|
||||
3.5 (ApplyBALAndComputeRoot with PathDB StateSet)
|
||||
↓
|
||||
3.6 (ProcessBlockWithBAL with trust model docs)
|
||||
↓
|
||||
3.7 (Reorg handling via BALHistory)
|
||||
↓
|
||||
3.8 (Wire into BlockChain)
|
||||
↓
|
||||
3.9 (Unit tests)
|
||||
↓
|
||||
3.10 (Integration test)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
**Pre-implementation (completed):**
|
||||
- [x] Code review completed for ApplyBALAndComputeRoot design
|
||||
- [x] BAL field names verified: `Accesses`, `ValueAfter` (from `core/types/bal/bal_encoding.go`)
|
||||
- [x] Commit ordering documented: storage tries before account trie
|
||||
- [x] PathDB StateSet construction researched and documented
|
||||
- [x] SELFDESTRUCT handling verified: tracked in BAL per EIP-7928
|
||||
- [x] Engine API delivery researched: standardized via engine_newPayloadV5, etc.
|
||||
|
||||
**After implementation:**
|
||||
- [ ] Master plan saved to `docs/partial-state/PARTIAL_STATEFULNESS_PLAN.md`
|
||||
- [ ] Phase 3 plan saved to `docs/partial-state/PHASE3_PLAN.md`
|
||||
- [ ] PartialState struct follows StateDB patterns
|
||||
- [ ] BAL hash verification works correctly
|
||||
- [ ] Balance/nonce/codeHash changes apply correctly for ALL accounts
|
||||
- [ ] Storage/code bytes stored only for tracked contracts
|
||||
- [ ] Commit ordering correct: storage trie commit → update account.Root → account trie commit
|
||||
- [ ] EIP-161 empty account deletion only for modified+empty+existed accounts
|
||||
- [ ] PathDB StateSet properly constructed with origins
|
||||
- [ ] Computed state root matches header
|
||||
- [ ] Reorg handling works (via blocks, not separate BALHistory)
|
||||
- [ ] All unit tests pass
|
||||
- [ ] Integration test passes
|
||||
|
||||
---
|
||||
|
||||
## Local Testing Strategy
|
||||
|
||||
### 1. Unit Test Execution
|
||||
```bash
|
||||
go test ./core/state/partial/... -v
|
||||
go test ./core/rawdb/... -run TestBAL -v
|
||||
```
|
||||
|
||||
### 2. Build Verification
|
||||
```bash
|
||||
go build ./...
|
||||
go build ./cmd/geth
|
||||
```
|
||||
|
||||
### 3. Integration Test
|
||||
```bash
|
||||
go test ./core/... -run TestPartialBlock -v -timeout 5m
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Open Items
|
||||
|
||||
1. **Engine API Integration**: BAL delivery is **already standardized** via extended Engine API methods:
|
||||
- `engine_newPayloadV5`: Validates computed access lists match provided BAL
|
||||
- `engine_getPayloadV6`: Returns `ExecutionPayloadV4` containing RLP-encoded BAL
|
||||
- `engine_getPayloadBodiesByHashV2` / `engine_getPayloadBodiesByRangeV2`: Retrieve historical BALs
|
||||
- **Status**: No additional design needed - use existing Engine API
|
||||
|
||||
---
|
||||
|
||||
## Critical Invariants
|
||||
|
||||
1. **State root must match**: Computed root from BAL application MUST match header's stateRoot
|
||||
2. **BAL hash verification**: Always verify BAL hash before processing
|
||||
3. **Account trie complete**: All account changes apply (balance, nonce, codeHash); only storage/code bytes are filtered for untracked
|
||||
4. **No execution required**: Block processing uses only BAL data, never re-executes transactions
|
||||
5. **Commit ordering**: Storage tries MUST be committed BEFORE account trie (storage roots needed first)
|
||||
6. **EIP-161 compliance**: Only delete accounts that were modified AND are now empty AND previously existed
|
||||
7. **BAL field names**: Use `Accesses` (not `Writes`) and `ValueAfter` (not `Value`) per `core/types/bal/bal_encoding.go`
|
||||
8. **PathDB StateSet**: Must construct proper `triedb.StateSet` with accounts/storage and their origins for `trieDB.Update()`
|
||||
|
||||
## Design Decisions
|
||||
|
||||
1. **SELFDESTRUCT is tracked**: Per EIP-7928, "Accounts destroyed within a transaction MUST be included in AccountChanges without nonce or code changes." Self-destructed accounts appear in BAL with balance changes but no nonce/code changes.
|
||||
|
||||
2. **Code handling for tracked vs untracked contracts**:
|
||||
- **All accounts**: Update `CodeHash` in account trie (required for correct state root)
|
||||
- **Tracked contracts only**: Store actual code bytes via `rawdb.WriteCode()`
|
||||
- **Untracked contracts**: Skip storing code bytes (saves storage, code not needed for partial state)
|
||||
|
||||
3. **Block attestation trust model** (Post-Merge architecture):
|
||||
- **CL responsibility**: Proposer signatures, sync committee attestations (2/3+ threshold), finality proofs, consensus rules
|
||||
- **EL responsibility**: Transaction execution, state root computation, receipt validation
|
||||
- **Trust boundary**: Blocks via Engine API (`engine_newPayloadV5`) are pre-attested by CL; EL trusts CL for consensus
|
||||
- **Partial state nodes**: Receive blocks via Engine API, so attestations are already verified
|
||||
- **Light client sync** (future): If blocks come from untrusted sources, use `beacon/light/CommitteeChain.VerifySignedHeader()`
|
||||
|
|
@ -291,6 +291,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
|
|||
options.PartialStateBALRetention = config.PartialState.BALRetention
|
||||
options.PartialStateChainRetention = config.PartialState.ChainRetention
|
||||
options.SnapshotNoBuild = true
|
||||
config.LogNoHistory = true // Partial state nodes have no receipts — disable log indexing
|
||||
}
|
||||
|
||||
eth.blockchain, err = core.NewBlockChain(chainDb, config.Genesis, eth.engine, options)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ import (
|
|||
"github.com/ethereum/go-ethereum/core"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/core/types/bal"
|
||||
"github.com/ethereum/go-ethereum/eth"
|
||||
"github.com/ethereum/go-ethereum/eth/ethconfig"
|
||||
"github.com/ethereum/go-ethereum/internal/telemetry"
|
||||
|
|
@ -883,6 +882,12 @@ func (api *ConsensusAPI) newPayload(ctx context.Context, params engine.Executabl
|
|||
}
|
||||
processingTime := time.Since(start)
|
||||
|
||||
// Write block to DB so ForkchoiceUpdated can find it via GetBlockByHash.
|
||||
// This writes header + body + BAL without requiring receipts or full state.
|
||||
if err := api.eth.BlockChain().WriteBlockWithoutState(block); err != nil {
|
||||
return api.invalid(err, parent.Header()), nil
|
||||
}
|
||||
|
||||
// Store BAL in history for potential reorg handling
|
||||
if history := api.eth.BlockChain().PartialState().History(); history != nil {
|
||||
history.Store(block.NumberU64(), params.BlockAccessList)
|
||||
|
|
|
|||
178
scripts/partial-state-devnet-test.sh
Executable file
178
scripts/partial-state-devnet-test.sh
Executable file
|
|
@ -0,0 +1,178 @@
|
|||
#!/bin/bash
|
||||
# Partial State Devnet Test Script
|
||||
#
|
||||
# This script sets up a 2-node devnet to test partial state functionality.
|
||||
# It starts a full node in dev mode and a partial state node that syncs from it.
|
||||
#
|
||||
# Usage: ./scripts/partial-state-devnet-test.sh
|
||||
|
||||
set -e
|
||||
|
||||
# Configuration
|
||||
FULL_NODE_DIR="/tmp/partial-state-test/full-node"
|
||||
PARTIAL_NODE_DIR="/tmp/partial-state-test/partial-node"
|
||||
FULL_NODE_PORT=30303
|
||||
PARTIAL_NODE_PORT=30304
|
||||
FULL_NODE_RPC=8545
|
||||
PARTIAL_NODE_RPC=8546
|
||||
|
||||
# Test contract address (will be tracked by partial node)
|
||||
TRACKED_CONTRACT="0x1234567890123456789012345678901234567890"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log_info() {
|
||||
echo -e "${GREEN}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
log_info "Cleaning up..."
|
||||
if [ -n "$FULL_PID" ]; then
|
||||
kill $FULL_PID 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "$PARTIAL_PID" ]; then
|
||||
kill $PARTIAL_PID 2>/dev/null || true
|
||||
fi
|
||||
wait 2>/dev/null || true
|
||||
log_info "Cleanup complete"
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# Build geth if not already built
|
||||
if [ ! -f "./geth" ]; then
|
||||
log_info "Building geth..."
|
||||
# go build ./cmd/geth
|
||||
fi
|
||||
|
||||
# Clean up old test data
|
||||
log_info "Setting up test directories..."
|
||||
rm -rf /tmp/partial-state-test
|
||||
mkdir -p "$FULL_NODE_DIR" "$PARTIAL_NODE_DIR"
|
||||
|
||||
# Start full node
|
||||
log_info "Starting full node..."
|
||||
./geth --datadir "$FULL_NODE_DIR" \
|
||||
--dev \
|
||||
--dev.period 2 \
|
||||
--port $FULL_NODE_PORT \
|
||||
--http --http.port $FULL_NODE_RPC \
|
||||
--http.api eth,net,web3,admin \
|
||||
--verbosity 2 \
|
||||
--nodiscover &
|
||||
FULL_PID=$!
|
||||
|
||||
log_info "Full node started with PID $FULL_PID"
|
||||
|
||||
# Wait for full node to start
|
||||
log_info "Waiting for full node to initialize..."
|
||||
sleep 5
|
||||
|
||||
# Get enode from full node
|
||||
log_info "Getting enode from full node..."
|
||||
for i in {1..10}; do
|
||||
ENODE=$(./geth attach "$FULL_NODE_DIR/geth.ipc" --exec admin.nodeInfo.enode 2>/dev/null | tr -d '"')
|
||||
if [ -n "$ENODE" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ -z "$ENODE" ]; then
|
||||
log_error "Failed to get enode from full node"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "Full node enode: ${ENODE:0:50}..."
|
||||
|
||||
# Start partial state node
|
||||
log_info "Starting partial state node..."
|
||||
./geth --datadir "$PARTIAL_NODE_DIR" \
|
||||
--port $PARTIAL_NODE_PORT \
|
||||
--http --http.port $PARTIAL_NODE_RPC \
|
||||
--http.api eth,net,web3 \
|
||||
--partial-state \
|
||||
--partial-state.contracts "$TRACKED_CONTRACT" \
|
||||
--bootnodes "$ENODE" \
|
||||
--networkid 1337 \
|
||||
--verbosity 2 &
|
||||
PARTIAL_PID=$!
|
||||
|
||||
log_info "Partial state node started with PID $PARTIAL_PID"
|
||||
|
||||
# Wait for nodes to connect
|
||||
log_info "Waiting for nodes to connect..."
|
||||
sleep 10
|
||||
|
||||
# Run tests
|
||||
log_info "Running tests..."
|
||||
|
||||
# Test 1: Check both nodes are running
|
||||
log_info "Test 1: Checking node connectivity..."
|
||||
FULL_PEERS=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:$FULL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
PARTIAL_PEERS=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"net_peerCount","params":[],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:$PARTIAL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
|
||||
log_info "Full node peers: $FULL_PEERS, Partial node peers: $PARTIAL_PEERS"
|
||||
|
||||
# Test 2: Send a transaction and verify sync
|
||||
log_info "Test 2: Sending test transaction..."
|
||||
./geth attach "$FULL_NODE_DIR/geth.ipc" --exec "eth.sendTransaction({from: eth.coinbase, to: '$TRACKED_CONTRACT', value: web3.toWei(1, 'ether')})" 2>/dev/null || true
|
||||
|
||||
# Wait for block to be mined
|
||||
sleep 5
|
||||
|
||||
# Test 3: Compare block numbers
|
||||
log_info "Test 3: Comparing block numbers..."
|
||||
FULL_BLOCK=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:$FULL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
PARTIAL_BLOCK=$(curl -s -X POST --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
-H "Content-Type: application/json" localhost:$PARTIAL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
|
||||
log_info "Full node block: $FULL_BLOCK, Partial node block: $PARTIAL_BLOCK"
|
||||
|
||||
# Test 4: Compare balances
|
||||
log_info "Test 4: Comparing account balances..."
|
||||
FULL_BALANCE=$(curl -s -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBalance\",\"params\":[\"$TRACKED_CONTRACT\",\"latest\"],\"id\":1}" \
|
||||
-H "Content-Type: application/json" localhost:$FULL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
PARTIAL_BALANCE=$(curl -s -X POST --data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBalance\",\"params\":[\"$TRACKED_CONTRACT\",\"latest\"],\"id\":1}" \
|
||||
-H "Content-Type: application/json" localhost:$PARTIAL_NODE_RPC | grep -o '"result":"[^"]*"' | cut -d'"' -f4)
|
||||
|
||||
log_info "Full node balance: $FULL_BALANCE, Partial node balance: $PARTIAL_BALANCE"
|
||||
|
||||
if [ "$FULL_BALANCE" = "$PARTIAL_BALANCE" ]; then
|
||||
log_info "Balances match!"
|
||||
else
|
||||
log_warn "Balances do not match (this may be expected if partial node is still syncing)"
|
||||
fi
|
||||
|
||||
# Summary
|
||||
echo ""
|
||||
log_info "========== Test Summary =========="
|
||||
log_info "Full node: PID=$FULL_PID, Port=$FULL_NODE_PORT, RPC=$FULL_NODE_RPC"
|
||||
log_info "Partial node: PID=$PARTIAL_PID, Port=$PARTIAL_NODE_PORT, RPC=$PARTIAL_NODE_RPC"
|
||||
log_info "Tracked contract: $TRACKED_CONTRACT"
|
||||
log_info ""
|
||||
log_info "Database sizes:"
|
||||
du -sh "$FULL_NODE_DIR/geth/chaindata" 2>/dev/null || echo " Full node: N/A"
|
||||
du -sh "$PARTIAL_NODE_DIR/geth/chaindata" 2>/dev/null || echo " Partial node: N/A"
|
||||
log_info "================================="
|
||||
echo ""
|
||||
|
||||
log_info "Test complete. Press Ctrl+C to stop nodes and cleanup."
|
||||
|
||||
# Keep running until interrupted
|
||||
wait
|
||||
15
scripts/partial-sync/contracts.json
Normal file
15
scripts/partial-sync/contracts.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"version": 1,
|
||||
"contracts": [
|
||||
{
|
||||
"address": "0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2",
|
||||
"name": "WETH9",
|
||||
"comment": "Wrapped Ether"
|
||||
},
|
||||
{
|
||||
"address": "0x6B175474E89094C44Da98b954EedeAC495271d0F",
|
||||
"name": "DAI",
|
||||
"comment": "Dai Stablecoin"
|
||||
}
|
||||
]
|
||||
}
|
||||
133
scripts/partial-sync/start_partial_sync.sh
Executable file
133
scripts/partial-sync/start_partial_sync.sh
Executable file
|
|
@ -0,0 +1,133 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# start_partial_sync.sh - Start a partial state sync on Ethereum mainnet.
|
||||
#
|
||||
# This script builds geth, generates a JWT secret, and starts geth in partial
|
||||
# state mode tracking only WETH and DAI contracts. After starting geth, you
|
||||
# must also start a Consensus Layer client (instructions printed at the end).
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
GETH_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
DATADIR="$HOME/.ethereum-partial-test"
|
||||
CONTRACTS_FILE="$SCRIPT_DIR/contracts.json"
|
||||
JWT_FILE="$DATADIR/jwt.hex"
|
||||
LOG_FILE="$DATADIR/geth.log"
|
||||
|
||||
echo "=== Partial State Sync Setup ==="
|
||||
echo "Geth source: $GETH_DIR"
|
||||
echo "Data directory: $DATADIR"
|
||||
echo "Contracts file: $CONTRACTS_FILE"
|
||||
echo ""
|
||||
|
||||
# Step 1: Always rebuild geth from current source to ensure fixes are included
|
||||
echo "Building geth from source at $GETH_DIR ..."
|
||||
cd "$GETH_DIR"
|
||||
go build -o build/bin/geth ./cmd/geth
|
||||
GETH="$GETH_DIR/build/bin/geth"
|
||||
echo "Built: $GETH"
|
||||
echo "Binary hash: $(shasum -a 256 "$GETH" | cut -d' ' -f1)"
|
||||
echo ""
|
||||
|
||||
# Step 2: Create datadir if needed
|
||||
mkdir -p "$DATADIR"
|
||||
|
||||
# Step 3: Generate JWT secret (if not exists)
|
||||
if [ ! -f "$JWT_FILE" ]; then
|
||||
echo "Generating JWT secret..."
|
||||
openssl rand -hex 32 > "$JWT_FILE"
|
||||
echo "JWT secret: $JWT_FILE"
|
||||
else
|
||||
echo "JWT secret already exists: $JWT_FILE"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Step 4: Verify contracts file exists
|
||||
if [ ! -f "$CONTRACTS_FILE" ]; then
|
||||
echo "ERROR: Contracts file not found: $CONTRACTS_FILE"
|
||||
exit 1
|
||||
fi
|
||||
echo "Tracked contracts:"
|
||||
cat "$CONTRACTS_FILE" | python3 -c "
|
||||
import json, sys
|
||||
data = json.load(sys.stdin)
|
||||
for c in data['contracts']:
|
||||
print(f\" {c['name']:10s} {c['address']}\")
|
||||
" 2>/dev/null || cat "$CONTRACTS_FILE"
|
||||
echo ""
|
||||
|
||||
# Step 5: Start geth
|
||||
echo "Starting geth in partial state mode..."
|
||||
echo "Log file: $LOG_FILE"
|
||||
echo ""
|
||||
|
||||
"$GETH" \
|
||||
--mainnet \
|
||||
--syncmode snap \
|
||||
--partial-state \
|
||||
--partial-state.contracts-file "$CONTRACTS_FILE" \
|
||||
--partial-state.bal-retention 256 \
|
||||
--partial-state.chain-retention 1024 \
|
||||
--history.logs.disable \
|
||||
--datadir "$DATADIR" \
|
||||
--authrpc.jwtsecret "$JWT_FILE" \
|
||||
--http \
|
||||
--http.api eth,net,web3,debug \
|
||||
--http.addr 127.0.0.1 \
|
||||
--http.port 8545 \
|
||||
--authrpc.addr 127.0.0.1 \
|
||||
--authrpc.port 8551 \
|
||||
--verbosity 3 \
|
||||
--log.file "$LOG_FILE" \
|
||||
&
|
||||
|
||||
GETH_PID=$!
|
||||
echo "Geth started (PID: $GETH_PID)"
|
||||
echo ""
|
||||
|
||||
# Step 6: Print CL instructions
|
||||
cat <<'INSTRUCTIONS'
|
||||
========================================
|
||||
NEXT STEP: Start a Consensus Layer client
|
||||
========================================
|
||||
|
||||
Geth (Execution Layer) is running. You now need a Consensus Layer client.
|
||||
Lighthouse is recommended. Install it from:
|
||||
|
||||
https://lighthouse-book.sigmaprime.io/installation.html
|
||||
|
||||
Then run (in a new terminal):
|
||||
|
||||
INSTRUCTIONS
|
||||
|
||||
echo " lighthouse bn \\"
|
||||
echo " --network mainnet \\"
|
||||
echo " --checkpoint-sync-url https://mainnet.checkpoint.sigp.io \\"
|
||||
echo " --execution-endpoint http://localhost:8551 \\"
|
||||
echo " --execution-jwt $JWT_FILE \\"
|
||||
echo " --datadir $HOME/.lighthouse-partial-test \\"
|
||||
echo " --slots-per-restore-point 8192 \\"
|
||||
echo " --disable-deposit-contract-sync \\"
|
||||
echo " --prune-blobs true \\"
|
||||
echo " --disable-backfill-rate-limiting \\"
|
||||
echo " --disable-optimistic-finalized-sync"
|
||||
|
||||
cat <<'INSTRUCTIONS'
|
||||
|
||||
Monitor sync progress:
|
||||
tail -f ~/.ethereum-partial-test/geth.log | grep -i "partial\|syncing\|sync stats"
|
||||
|
||||
Check sync status via RPC:
|
||||
curl -s -X POST http://localhost:8545 \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' | jq
|
||||
|
||||
When sync completes, run verification:
|
||||
./scripts/partial-sync/verify_partial_sync.sh
|
||||
|
||||
========================================
|
||||
INSTRUCTIONS
|
||||
|
||||
# Wait for geth process
|
||||
wait $GETH_PID
|
||||
353
scripts/partial-sync/verify_partial_sync.sh
Executable file
353
scripts/partial-sync/verify_partial_sync.sh
Executable file
|
|
@ -0,0 +1,353 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# verify_partial_sync.sh - Verify partial state sync correctness.
|
||||
#
|
||||
# Runs JSON-RPC checks against a running geth node to verify:
|
||||
# 1. All accounts are accessible (full account trie synced)
|
||||
# 2. Tracked contract storage and code are present
|
||||
# 3. Untracked contract storage and code are correctly rejected
|
||||
#
|
||||
# Usage:
|
||||
# ./verify_partial_sync.sh # RPC checks (geth must be running)
|
||||
# ./verify_partial_sync.sh --db-only # Database inspection (geth must be stopped)
|
||||
# ./verify_partial_sync.sh --all # Both (stops geth for DB checks)
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
RPC_URL="${RPC_URL:-http://localhost:8545}"
|
||||
DATADIR="${DATADIR:-$HOME/.ethereum-partial-test}"
|
||||
GETH="${GETH:-$(dirname "${BASH_SOURCE[0]}")/../../build/bin/geth}"
|
||||
|
||||
# Tracked contracts (WETH, DAI)
|
||||
WETH="0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
|
||||
DAI="0x6B175474E89094C44Da98b954EedeAC495271d0F"
|
||||
|
||||
# Untracked contracts (USDC, Uniswap V2 Router)
|
||||
USDC="0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48"
|
||||
UNISWAP_ROUTER="0x7a250d5630B4cF539739dF2C5dAcb4c659F2488D"
|
||||
|
||||
# ERC20 totalSupply() selector
|
||||
TOTAL_SUPPLY="0x18160ddd"
|
||||
|
||||
# Counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
TOTAL=0
|
||||
|
||||
# ─── Helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
check_deps() {
|
||||
for cmd in curl jq; do
|
||||
if ! command -v "$cmd" &>/dev/null; then
|
||||
echo "ERROR: '$cmd' is required but not installed."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
rpc_call() {
|
||||
local method="$1"
|
||||
local params="$2"
|
||||
curl -s -X POST "$RPC_URL" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":$params,\"id\":1}"
|
||||
}
|
||||
|
||||
# Check that result field is non-zero hex
|
||||
check_nonzero() {
|
||||
local label="$1"
|
||||
local method="$2"
|
||||
local params="$3"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local response
|
||||
response=$(rpc_call "$method" "$params")
|
||||
|
||||
local error
|
||||
error=$(echo "$response" | jq -r '.error // empty')
|
||||
if [ -n "$error" ]; then
|
||||
echo " [FAIL] $label"
|
||||
echo " Error: $(echo "$response" | jq -r '.error.message')"
|
||||
FAIL=$((FAIL + 1))
|
||||
return
|
||||
fi
|
||||
|
||||
local result
|
||||
result=$(echo "$response" | jq -r '.result')
|
||||
|
||||
if [ "$result" = "0x0" ] || [ "$result" = "0x" ] || [ "$result" = "null" ] || [ -z "$result" ]; then
|
||||
echo " [FAIL] $label (got: $result)"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
# Truncate long results for display
|
||||
local display="$result"
|
||||
if [ ${#display} -gt 20 ]; then
|
||||
display="${display:0:20}..."
|
||||
fi
|
||||
echo " [PASS] $label ($display)"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Check that result is non-empty bytecode (not "0x")
|
||||
check_code() {
|
||||
local label="$1"
|
||||
local addr="$2"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local response
|
||||
response=$(rpc_call "eth_getCode" "[\"$addr\",\"latest\"]")
|
||||
|
||||
local error
|
||||
error=$(echo "$response" | jq -r '.error // empty')
|
||||
if [ -n "$error" ]; then
|
||||
echo " [FAIL] $label"
|
||||
echo " Error: $(echo "$response" | jq -r '.error.message')"
|
||||
FAIL=$((FAIL + 1))
|
||||
return
|
||||
fi
|
||||
|
||||
local result
|
||||
result=$(echo "$response" | jq -r '.result')
|
||||
local len=$(( (${#result} - 2) / 2 )) # bytes = (hex_len - "0x" prefix) / 2
|
||||
|
||||
if [ "$result" = "0x" ] || [ "$len" -le 0 ]; then
|
||||
echo " [FAIL] $label (empty code)"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
echo " [PASS] $label ($len bytes)"
|
||||
PASS=$((PASS + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Check that RPC returns a specific error code
|
||||
check_error() {
|
||||
local label="$1"
|
||||
local method="$2"
|
||||
local params="$3"
|
||||
local expected_code="$4"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local response
|
||||
response=$(rpc_call "$method" "$params")
|
||||
|
||||
local error_code
|
||||
error_code=$(echo "$response" | jq -r '.error.code // empty')
|
||||
|
||||
if [ "$error_code" = "$expected_code" ]; then
|
||||
local msg
|
||||
msg=$(echo "$response" | jq -r '.error.message')
|
||||
echo " [PASS] $label (error $error_code: $msg)"
|
||||
PASS=$((PASS + 1))
|
||||
elif [ -n "$error_code" ]; then
|
||||
echo " [FAIL] $label (expected error $expected_code, got $error_code)"
|
||||
FAIL=$((FAIL + 1))
|
||||
else
|
||||
local result
|
||||
result=$(echo "$response" | jq -r '.result')
|
||||
echo " [FAIL] $label (expected error $expected_code, but got result: ${result:0:20}...)"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Check that eth_call returns an error (any error)
|
||||
check_call_error() {
|
||||
local label="$1"
|
||||
local to="$2"
|
||||
local data="$3"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local response
|
||||
response=$(rpc_call "eth_call" "[{\"to\":\"$to\",\"data\":\"$data\"},\"latest\"]")
|
||||
|
||||
local error
|
||||
error=$(echo "$response" | jq -r '.error // empty')
|
||||
|
||||
if [ -n "$error" ]; then
|
||||
local msg
|
||||
msg=$(echo "$response" | jq -r '.error.message')
|
||||
echo " [PASS] $label (error: ${msg:0:50})"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
local result
|
||||
result=$(echo "$response" | jq -r '.result')
|
||||
echo " [FAIL] $label (expected error, got result: ${result:0:20}...)"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
# ─── RPC Verification ────────────────────────────────────────────────
|
||||
|
||||
run_rpc_checks() {
|
||||
echo "=== Partial State Sync Verification ==="
|
||||
echo ""
|
||||
echo "RPC endpoint: $RPC_URL"
|
||||
echo ""
|
||||
|
||||
# A. Sync Status
|
||||
echo "Sync Status:"
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local syncing
|
||||
syncing=$(rpc_call "eth_syncing" "[]" | jq -r '.result')
|
||||
if [ "$syncing" = "false" ]; then
|
||||
echo " [PASS] eth_syncing returns false"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " [WARN] eth_syncing returns: $syncing (sync may still be in progress)"
|
||||
echo " Some checks may fail until sync completes."
|
||||
PASS=$((PASS + 1)) # Not a failure, just a warning
|
||||
fi
|
||||
|
||||
TOTAL=$((TOTAL + 1))
|
||||
local block_hex
|
||||
block_hex=$(rpc_call "eth_blockNumber" "[]" | jq -r '.result')
|
||||
if [ -n "$block_hex" ] && [ "$block_hex" != "null" ]; then
|
||||
local block_dec
|
||||
block_dec=$(printf "%d" "$block_hex" 2>/dev/null || echo "?")
|
||||
echo " [PASS] Block number: $block_dec ($block_hex)"
|
||||
PASS=$((PASS + 1))
|
||||
else
|
||||
echo " [FAIL] Could not get block number"
|
||||
FAIL=$((FAIL + 1))
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# B. Account Data (all accounts - full trie synced)
|
||||
echo "Account Data (all accounts - full trie synced):"
|
||||
check_nonzero "USDC contract balance" "eth_getBalance" "[\"$USDC\",\"latest\"]"
|
||||
check_nonzero "WETH contract balance" "eth_getBalance" "[\"$WETH\",\"latest\"]"
|
||||
check_nonzero "Uniswap Router balance" "eth_getBalance" "[\"$UNISWAP_ROUTER\",\"latest\"]"
|
||||
check_nonzero "USDC nonce" "eth_getTransactionCount" "[\"$USDC\",\"latest\"]"
|
||||
echo ""
|
||||
|
||||
# C. Tracked Contracts (WETH, DAI)
|
||||
echo "Tracked Contracts (WETH, DAI):"
|
||||
check_code "WETH code" "$WETH"
|
||||
check_code "DAI code" "$DAI"
|
||||
check_nonzero "WETH storage slot 0x0" "eth_getStorageAt" "[\"$WETH\",\"0x0\",\"latest\"]"
|
||||
check_nonzero "DAI storage slot 0x0" "eth_getStorageAt" "[\"$DAI\",\"0x0\",\"latest\"]"
|
||||
check_nonzero "eth_call WETH.totalSupply()" "eth_call" "[{\"to\":\"$WETH\",\"data\":\"$TOTAL_SUPPLY\"},\"latest\"]"
|
||||
check_nonzero "eth_call DAI.totalSupply()" "eth_call" "[{\"to\":\"$DAI\",\"data\":\"$TOTAL_SUPPLY\"},\"latest\"]"
|
||||
echo ""
|
||||
|
||||
# D. Untracked Contracts (USDC, Uniswap V2 Router)
|
||||
echo "Untracked Contracts (USDC, Uniswap V2 Router):"
|
||||
check_error "USDC eth_getStorageAt" "eth_getStorageAt" "[\"$USDC\",\"0x0\",\"latest\"]" "-32001"
|
||||
check_error "Router eth_getStorageAt" "eth_getStorageAt" "[\"$UNISWAP_ROUTER\",\"0x0\",\"latest\"]" "-32001"
|
||||
check_error "USDC eth_getCode" "eth_getCode" "[\"$USDC\",\"latest\"]" "-32002"
|
||||
check_error "Router eth_getCode" "eth_getCode" "[\"$UNISWAP_ROUTER\",\"latest\"]" "-32002"
|
||||
check_call_error "eth_call USDC.totalSupply()" "$USDC" "$TOTAL_SUPPLY"
|
||||
echo ""
|
||||
|
||||
# Summary
|
||||
echo "========================================="
|
||||
if [ $FAIL -eq 0 ]; then
|
||||
echo " Results: $PASS/$TOTAL passed"
|
||||
else
|
||||
echo " Results: $PASS/$TOTAL passed, $FAIL FAILED"
|
||||
fi
|
||||
echo "========================================="
|
||||
}
|
||||
|
||||
# ─── Database Verification ───────────────────────────────────────────
|
||||
|
||||
run_db_checks() {
|
||||
echo ""
|
||||
echo "=== Database-Level Verification ==="
|
||||
echo ""
|
||||
echo "Data directory: $DATADIR"
|
||||
echo ""
|
||||
|
||||
# Check geth binary exists
|
||||
if [ ! -x "$GETH" ]; then
|
||||
echo "ERROR: geth binary not found at $GETH"
|
||||
echo "Set GETH env var or build first: go build -o build/bin/geth ./cmd/geth"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check datadir exists
|
||||
if [ ! -d "$DATADIR" ]; then
|
||||
echo "ERROR: Data directory not found: $DATADIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check geth is not running (LevelDB requires exclusive access)
|
||||
if pgrep -f "geth.*partial-test" > /dev/null 2>&1; then
|
||||
echo "WARNING: geth appears to be running. Stop it first for database inspection."
|
||||
echo " kill \$(pgrep -f 'geth.*partial-test')"
|
||||
echo ""
|
||||
fi
|
||||
|
||||
echo "Running: geth db inspect"
|
||||
echo "(this may take a while for large databases)"
|
||||
echo ""
|
||||
|
||||
"$GETH" db inspect --datadir "$DATADIR" 2>&1 | tee /tmp/partial-sync-inspect.txt
|
||||
|
||||
echo ""
|
||||
echo "Inspection output saved to: /tmp/partial-sync-inspect.txt"
|
||||
echo ""
|
||||
echo "What to check in the output above:"
|
||||
echo " - 'Account snapshot' : Should be large (~45 GiB) - full account trie"
|
||||
echo " - 'Storage snapshot' : Should be TINY (< 1 GiB) - only WETH + DAI"
|
||||
echo " - 'Contract codes' : Should be very small - only 2 contracts"
|
||||
echo " - 'Bodies' : Should be tiny (< 10 MiB) - chain retention=1024"
|
||||
echo " - 'Receipts' : Should be tiny (< 10 MiB) - chain retention=1024"
|
||||
echo " - 'Headers' : ~9 GiB (full chain, non-prunable)"
|
||||
echo " - Compare total DB size to a full node (~640+ GiB)"
|
||||
echo " - Expected total: ~59 GiB (headers + partial state)"
|
||||
echo ""
|
||||
|
||||
# Try dumptrie for tracked contract (WETH)
|
||||
echo "Verifying tracked contract storage (WETH)..."
|
||||
echo "Running: geth db dumptrie (limited to 5 entries)"
|
||||
echo ""
|
||||
|
||||
# Compute WETH account hash (keccak256 of address bytes)
|
||||
local weth_hash
|
||||
weth_hash=$(python3 -c "
|
||||
from hashlib import sha3_256
|
||||
addr = bytes.fromhex('C02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2')
|
||||
print('0x' + sha3_256(addr).hexdigest())
|
||||
" 2>/dev/null || echo "")
|
||||
|
||||
if [ -n "$weth_hash" ]; then
|
||||
echo "WETH account hash: $weth_hash"
|
||||
# Note: dumptrie requires state-root and storage-root which need the account data.
|
||||
# For now, just note the hash for manual inspection.
|
||||
echo "(Use 'geth db dumptrie <state-root> $weth_hash <storage-root> \"\" 5' for manual inspection)"
|
||||
else
|
||||
echo "Python3 not available for hash computation. Skipping dumptrie."
|
||||
fi
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ─── Main ────────────────────────────────────────────────────────────
|
||||
|
||||
check_deps
|
||||
|
||||
MODE="${1:-rpc}"
|
||||
|
||||
case "$MODE" in
|
||||
--db-only)
|
||||
run_db_checks
|
||||
;;
|
||||
--all)
|
||||
run_rpc_checks
|
||||
echo ""
|
||||
echo "Stopping geth for database inspection..."
|
||||
kill "$(pgrep -f 'geth.*partial-test')" 2>/dev/null || true
|
||||
sleep 3
|
||||
run_db_checks
|
||||
;;
|
||||
*)
|
||||
run_rpc_checks
|
||||
echo ""
|
||||
echo "For database-level verification, run:"
|
||||
echo " $0 --db-only (after stopping geth)"
|
||||
echo " $0 --all (stops geth automatically)"
|
||||
;;
|
||||
esac
|
||||
|
||||
exit $FAIL
|
||||
Loading…
Reference in a new issue