Merge pull request #1 from ethereum/master

test

.gitea/workflows/release-azure-cleanup.yml

@@ -0,0 +1,22 @@
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  azure-cleanup:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Run cleanup script
+      run: |
+        go run build/ci.go purge -store gethstore/builds -days 14
+      env:
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}

.gitea/workflows/release-ppa.yml

@@ -1,27 +1,14 @@
-name: Release Builds (cron)
-
 on:
   schedule:
-    cron: '0 0 * * *'
+    - cron: '0 16 * * *'
+  push:
+    branches:
+      - "release/*"
+  workflow_dispatch:
 
 jobs:
-  azure-cleanup:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Go
-      uses: actions/setup-go@v5
-      with:
-        go-version: 1.24
-        cache: false
-
-    - name: Run cleanup script
-      run: |
-        go run build/ci.go purge -store gethstore/builds -days 14
-
   ppa:
-    name: PPA Upload (master)
+    name: PPA Upload
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -34,9 +21,16 @@ jobs:
 
     - name: Install deb toolchain
       run: |
+        apt-get update
         apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
 
+    - name: Add launchpad to known_hosts
+      run: |
+        echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+
     - name: Run ci.go
       run: |
-        echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
         go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+      env:
+        PPA_SIGNING_KEY: ${{ secrets.PPA_SIGNING_KEY }}
+        PPA_SSH_KEY: ${{ secrets.PPA_SSH_KEY }}

.gitea/workflows/release.yml

@@ -1,11 +1,12 @@
-name: Release Builds (push)
-
 on:
   push:
-    branches: [ master ]
+    branches:
+      - "master"
+    tags:
+      - "v*"
 
 jobs:
-  linux:
+  linux-intel:
     name: Linux Build
     runs-on: ubuntu-latest
     steps:
@@ -17,13 +18,108 @@ jobs:
         go-version: 1.24
         cache: false
 
-    - name: display environment
+    - name: Install cross toolchain
       run: |
-        env
+        apt-get update
+        apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib
 
-    - name: Run build
+    - name: Build (amd64)
       run: |
-        go run build/ci.go install
+        go run build/ci.go install -arch amd64 -dlgo
+
+    - name: Create/upload archive (amd64)
+      run: |
+        go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -f build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Build (386)
+      run: |
+        go run build/ci.go install -arch 386 -dlgo
+
+    - name: Create/upload archive (386)
+      run: |
+        go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -f build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+  linux-arm:
+    name: Linux Build (arm)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Install cross toolchain
+      run: |
+        apt-get update
+        apt-get -yq --no-install-suggests --no-install-recommends install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+        ln -s /usr/include/asm-generic /usr/include/asm
+
+    - name: Build (arm64)
+      run: |
+        go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
+
+    - name: Create/upload archive (arm64)
+      run: |
+        go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm5)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "5"
+
+    - name: Create/upload archive (arm5)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+      env:
+        GOARM: "5"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm6)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "6"
+
+    - name: Create/upload archive (arm6)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        GOARM: "6"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm7)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "7"
+
+    - name: Create/upload archive (arm7)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        GOARM: "7"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
 
   docker:
     name: Docker Image
@@ -44,5 +140,8 @@ jobs:
         cache: false
 
     - name: Run docker build
+      env:
+        DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
       run: |
-        go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64
+        go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 -upload

cmd/geth/config.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/flags"
@@ -180,6 +181,45 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	return stack, cfg
 }
 
+// constructs the disclaimer text block which will be printed in the logs upon
+// startup when Geth is running in dev mode.
+func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string {
+	devModeBanner := `You are running Geth in --dev mode. Please note the following:
+
+  1. This mode is only intended for fast, iterative development without assumptions on
+     security or persistence.
+  2. The database is created in memory unless specified otherwise. Therefore, shutting down
+     your computer or losing power will wipe your entire block data and chain state for
+     your dev environment.
+  3. A random, pre-allocated developer account will be available and unlocked as
+     eth.coinbase, which can be used for testing. The random dev account is temporary,
+     stored on a ramdisk, and will be lost if your machine is restarted.
+  4. Mining is enabled by default. However, the client will only seal blocks if transactions
+     are pending in the mempool. The miner's minimum accepted gas price is 1.
+  5. Networking is disabled; there is no listen-address, the maximum number of peers is set
+     to 0, and discovery is disabled.
+`
+	if !ctx.IsSet(utils.DataDirFlag.Name) {
+		devModeBanner += fmt.Sprintf(`
+
+ Running in ephemeral mode.  The following account has been prefunded in the genesis:
+
+       Account
+       ------------------
+       0x%x (10^49 ETH)
+`, cfg.Eth.Miner.PendingFeeRecipient)
+		if cfg.Eth.Miner.PendingFeeRecipient == utils.DeveloperAddr {
+			devModeBanner += fmt.Sprintf(` 
+       Private Key
+       ------------------
+       0x%x
+`, crypto.FromECDSA(utils.DeveloperKey))
+		}
+	}
+
+	return devModeBanner
+}
+
 // makeFullNode loads geth configuration and creates the Ethereum backend.
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
@@ -239,6 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		}
 		catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon)
 		stack.RegisterLifecycle(simBeacon)
+
+		banner := constructDevModeBanner(ctx, cfg)
+		for _, line := range strings.Split(banner, "\n") {
+			log.Warn(line)
+		}
 	} else if ctx.IsSet(utils.BeaconApiFlag.Name) {
 		// Start blsync mode.
 		srv := rpc.NewServer()

cmd/geth/main.go

@@ -294,24 +294,6 @@ func prepare(ctx *cli.Context) {
 	case ctx.IsSet(utils.HoodiFlag.Name):
 		log.Info("Starting Geth on Hoodi testnet...")
 
-	case ctx.IsSet(utils.DeveloperFlag.Name):
-		log.Info("Starting Geth in ephemeral dev mode...")
-		log.Warn(`You are running Geth in --dev mode. Please note the following:
-
-  1. This mode is only intended for fast, iterative development without assumptions on
-     security or persistence.
-  2. The database is created in memory unless specified otherwise. Therefore, shutting down
-     your computer or losing power will wipe your entire block data and chain state for
-     your dev environment.
-  3. A random, pre-allocated developer account will be available and unlocked as
-     eth.coinbase, which can be used for testing. The random dev account is temporary,
-     stored on a ramdisk, and will be lost if your machine is restarted.
-  4. Mining is enabled by default. However, the client will only seal blocks if transactions
-     are pending in the mempool. The miner's minimum accepted gas price is 1.
-  5. Networking is disabled; there is no listen-address, the maximum number of peers is set
-     to 0, and discovery is disabled.
-`)
-
 	case !ctx.IsSet(utils.NetworkIdFlag.Name):
 		log.Info("Starting Geth on Ethereum mainnet...")
 	}

cmd/utils/flags.go

@@ -984,6 +984,12 @@ var (
 	}
 )
 
+// default account to prefund when running Geth in dev mode
+var (
+	DeveloperKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	DeveloperAddr   = crypto.PubkeyToAddress(DeveloperKey.PublicKey)
+)
+
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -1769,9 +1775,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		} else if accs := ks.Accounts(); len(accs) > 0 {
 			developer = ks.Accounts()[0]
 		} else {
-			developer, err = ks.NewAccount(passphrase)
+			developer, err = ks.ImportECDSA(DeveloperKey, passphrase)
 			if err != nil {
-				Fatalf("Failed to create developer account: %v", err)
+				Fatalf("Failed to import developer account: %v", err)
 			}
 		}
 		// Make sure the address is configured as fee recipient, otherwise
@@ -1786,14 +1792,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		log.Info("Using developer account", "address", developer.Address)
 
-		// Create a new developer genesis block or reuse existing one
+		// configure default developer genesis which will be used unless a
+		// datadir is specified and a chain is preexisting at that location.
 		cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), &developer.Address)
+
+		// If a datadir is specified, ensure that any preexisting chain in that location
+		// has a configuration that is compatible with dev mode: it must be merged at genesis.
 		if ctx.IsSet(DataDirFlag.Name) {
 			chaindb := tryMakeReadOnlyDatabase(ctx, stack)
 			if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
-				cfg.Genesis = nil // fallback to db content
+				// signal fallback to preexisting chain on disk
+				cfg.Genesis = nil
 
-				//validate genesis has PoS enabled in block 0
 				genesis, err := core.ReadGenesis(chaindb)
 				if err != nil {
 					Fatalf("Could not read genesis from database: %v", err)

triedb/pathdb/database.go

@@ -487,7 +487,7 @@ func (db *Database) Enable(root common.Hash) error {
 
 	// Re-construct a new disk layer backed by persistent state
 	// and schedule the state snapshot generation if it's permitted.
-	db.tree.reset(generateSnapshot(db, root, db.isVerkle || db.config.SnapshotNoBuild))
+	db.tree.init(generateSnapshot(db, root, db.isVerkle || db.config.SnapshotNoBuild))
 	log.Info("Rebuilt trie database", "root", root)
 	return nil
 }
@@ -529,7 +529,7 @@ func (db *Database) Recover(root common.Hash) error {
 		// reset layer with newly created disk layer. It must be
 		// done after each revert operation, otherwise the new
 		// disk layer won't be accessible from outside.
-		db.tree.reset(dl)
+		db.tree.init(dl)
 	}
 	rawdb.DeleteTrieJournal(db.diskdb)
 

triedb/pathdb/difflayer.go

@@ -156,7 +156,7 @@ func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes *no
 }
 
 // persist flushes the diff layer and all its parent layers to disk layer.
-func (dl *diffLayer) persist(force bool) (layer, error) {
+func (dl *diffLayer) persist(force bool) (*diskLayer, error) {
 	if parent, ok := dl.parentLayer().(*diffLayer); ok {
 		// Hold the lock to prevent any read operation until the new
 		// parent is linked correctly.
@@ -183,7 +183,7 @@ func (dl *diffLayer) size() uint64 {
 
 // diffToDisk merges a bottom-most diff into the persistent disk layer underneath
 // it. The method will panic if called onto a non-bottom-most diff layer.
-func diffToDisk(layer *diffLayer, force bool) (layer, error) {
+func diffToDisk(layer *diffLayer, force bool) (*diskLayer, error) {
 	disk, ok := layer.parentLayer().(*diskLayer)
 	if !ok {
 		panic(fmt.Sprintf("unknown layer type: %T", layer.parentLayer()))

triedb/pathdb/disklayer.go

@@ -94,15 +94,6 @@ func (dl *diskLayer) setGenerator(generator *generator) {
 	dl.generator = generator
 }
 
-// isStale return whether this layer has become stale (was flattened across) or if
-// it's still live.
-func (dl *diskLayer) isStale() bool {
-	dl.lock.RLock()
-	defer dl.lock.RUnlock()
-
-	return dl.stale
-}
-
 // markStale sets the stale flag as true.
 func (dl *diskLayer) markStale() {
 	dl.lock.Lock()

triedb/pathdb/layertree.go

@@ -31,29 +31,50 @@ import (
 // thread-safe to use. However, callers need to ensure the thread-safety
 // of the referenced layer by themselves.
 type layerTree struct {
-	lock   sync.RWMutex
+	base   *diskLayer
 	layers map[common.Hash]layer
+
+	// descendants is a two-dimensional map where the keys represent
+	// an ancestor state root, and the values are the state roots of
+	// all its descendants.
+	//
+	// For example: r -> [c1, c2, ..., cn], where c1 through cn are
+	// the descendants of state r.
+	//
+	// This map includes all the existing diff layers and the disk layer.
+	descendants map[common.Hash]map[common.Hash]struct{}
+	lookup      *lookup
+	lock        sync.RWMutex
 }
 
 // newLayerTree constructs the layerTree with the given head layer.
 func newLayerTree(head layer) *layerTree {
 	tree := new(layerTree)
-	tree.reset(head)
+	tree.init(head)
 	return tree
 }
 
-// reset initializes the layerTree by the given head layer.
+// init initializes the layerTree by the given head layer.
-// All the ancestors will be iterated out and linked in the tree.
+func (tree *layerTree) init(head layer) {
-func (tree *layerTree) reset(head layer) {
 	tree.lock.Lock()
 	defer tree.lock.Unlock()
 
-	var layers = make(map[common.Hash]layer)
+	current := head
-	for head != nil {
+	tree.layers = make(map[common.Hash]layer)
-		layers[head.rootHash()] = head
+	tree.descendants = make(map[common.Hash]map[common.Hash]struct{})
-		head = head.parentLayer()
+
+	for {
+		tree.layers[current.rootHash()] = current
+		tree.fillAncestors(current)
+
+		parent := current.parentLayer()
+		if parent == nil {
+			break
 		}
-	tree.layers = layers
+		current = parent
+	}
+	tree.base = current.(*diskLayer) // panic if it's not a disk layer
+	tree.lookup = newLookup(head, tree.isDescendant)
 }
 
 // get retrieves a layer belonging to the given state root.
@@ -64,6 +85,43 @@ func (tree *layerTree) get(root common.Hash) layer {
 	return tree.layers[root]
 }
 
+// isDescendant returns whether the specified layer with given root is a
+// descendant of a specific ancestor.
+//
+// This function assumes the read lock has been held.
+func (tree *layerTree) isDescendant(root common.Hash, ancestor common.Hash) bool {
+	subset := tree.descendants[ancestor]
+	if subset == nil {
+		return false
+	}
+	_, ok := subset[root]
+	return ok
+}
+
+// fillAncestors identifies the ancestors of the given layer and populates the
+// descendants set. The ancestors include the diff layers below the supplied
+// layer and also the disk layer.
+//
+// This function assumes the write lock has been held.
+func (tree *layerTree) fillAncestors(layer layer) {
+	hash := layer.rootHash()
+	for {
+		parent := layer.parentLayer()
+		if parent == nil {
+			break
+		}
+		layer = parent
+
+		phash := parent.rootHash()
+		subset := tree.descendants[phash]
+		if subset == nil {
+			subset = make(map[common.Hash]struct{})
+			tree.descendants[phash] = subset
+		}
+		subset[hash] = struct{}{}
+	}
+}
+
 // forEach iterates the stored layers inside and applies the
 // given callback on them.
 func (tree *layerTree) forEach(onLayer func(layer)) {
@@ -101,8 +159,16 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6
 	l := parent.update(root, parent.stateID()+1, block, newNodeSet(nodes.Flatten()), states)
 
 	tree.lock.Lock()
+	defer tree.lock.Unlock()
+
+	// Link the given layer into the layer set
 	tree.layers[l.rootHash()] = l
-	tree.lock.Unlock()
+
+	// Link the given layer into its ancestors (up to the current disk layer)
+	tree.fillAncestors(l)
+
+	// Link the given layer into the state mutation history
+	tree.lookup.addLayer(l)
 	return nil
 }
 
@@ -127,8 +193,16 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		if err != nil {
 			return err
 		}
-		// Replace the entire layer tree with the flat base
+		tree.base = base
-		tree.layers = map[common.Hash]layer{base.rootHash(): base}
+
+		// Reset the layer tree with the single new disk layer
+		tree.layers = map[common.Hash]layer{
+			base.rootHash(): base,
+		}
+		// Resets the descendants map, since there's only a single disk layer
+		// with no descendants.
+		tree.descendants = make(map[common.Hash]map[common.Hash]struct{})
+		tree.lookup = newLookup(base, tree.isDescendant)
 		return nil
 	}
 	// Dive until we run out of layers or reach the persistent database
@@ -143,6 +217,11 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 	}
 	// We're out of layers, flatten anything below, stopping if it's the disk or if
 	// the memory limit is not yet exceeded.
+	var (
+		err      error
+		replaced layer
+		newBase  *diskLayer
+	)
 	switch parent := diff.parentLayer().(type) {
 	case *diskLayer:
 		return nil
@@ -152,14 +231,33 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 		// parent is linked correctly.
 		diff.lock.Lock()
 
-		base, err := parent.persist(false)
+		// Hold the reference of the original layer being replaced
+		replaced = parent
+
+		// Replace the original parent layer with new disk layer. The procedure
+		// can be illustrated as below:
+		//
+		// Before change:
+		//     Chain:
+		//        C1->C2->C3->C4 (HEAD)
+		//          ->C2'->C3'->C4'
+		//
+		// After change:
+		//     Chain:
+		//        (a) C3->C4 (HEAD)
+		//		  (b) C1->C2
+		//		        ->C2'->C3'->C4'
+		// The original C3 is replaced by the new base (with root C3)
+		// Dangling layers in (b) will be removed later
+		newBase, err = parent.persist(false)
 		if err != nil {
 			diff.lock.Unlock()
 			return err
 		}
-		tree.layers[base.rootHash()] = base
+		tree.layers[newBase.rootHash()] = newBase
-		diff.parent = base
 
+		// Link the new parent and release the lock
+		diff.parent = newBase
 		diff.lock.Unlock()
 
 	default:
@@ -173,19 +271,28 @@ func (tree *layerTree) cap(root common.Hash, layers int) error {
 			children[parent] = append(children[parent], root)
 		}
 	}
+	clearDiff := func(layer layer) {
+		diff, ok := layer.(*diffLayer)
+		if !ok {
+			return
+		}
+		tree.lookup.removeLayer(diff)
+	}
 	var remove func(root common.Hash)
 	remove = func(root common.Hash) {
+		clearDiff(tree.layers[root])
+
+		// Unlink the layer from the layer tree and cascade to its children
+		delete(tree.descendants, root)
 		delete(tree.layers, root)
 		for _, child := range children[root] {
 			remove(child)
 		}
 		delete(children, root)
 	}
-	for root, layer := range tree.layers {
+	remove(tree.base.rootHash()) // remove the old/stale disk layer
-		if dl, ok := layer.(*diskLayer); ok && dl.isStale() {
+	clearDiff(replaced)          // remove the lookup data of the stale parent being replaced
-			remove(root)
+	tree.base = newBase          // update the base layer with newly constructed one
-		}
-	}
 	return nil
 }
 
@@ -194,17 +301,41 @@ func (tree *layerTree) bottom() *diskLayer {
 	tree.lock.RLock()
 	defer tree.lock.RUnlock()
 
-	if len(tree.layers) == 0 {
+	return tree.base
-		return nil // Shouldn't happen, empty tree
 }
-	// pick a random one as the entry point
+
-	var current layer
+// lookupAccount returns the layer that is guaranteed to contain the account data
-	for _, layer := range tree.layers {
+// corresponding to the specified state root being queried.
-		current = layer
+func (tree *layerTree) lookupAccount(accountHash common.Hash, state common.Hash) (layer, error) {
-		break
+	// Hold the read lock to prevent the unexpected layer changes
+	tree.lock.RLock()
+	defer tree.lock.RUnlock()
+
+	tip := tree.lookup.accountTip(accountHash, state, tree.base.root)
+	if tip == (common.Hash{}) {
+		return nil, fmt.Errorf("[%#x] %w", state, errSnapshotStale)
 	}
-	for current.parentLayer() != nil {
+	l := tree.layers[tip]
-		current = current.parentLayer()
+	if l == nil {
+		return nil, fmt.Errorf("triedb layer [%#x] missing", tip)
 	}
-	return current.(*diskLayer)
+	return l, nil
+}
+
+// lookupStorage returns the layer that is guaranteed to contain the storage slot
+// data corresponding to the specified state root being queried.
+func (tree *layerTree) lookupStorage(accountHash common.Hash, slotHash common.Hash, state common.Hash) (layer, error) {
+	// Hold the read lock to prevent the unexpected layer changes
+	tree.lock.RLock()
+	defer tree.lock.RUnlock()
+
+	tip := tree.lookup.storageTip(accountHash, slotHash, state, tree.base.root)
+	if tip == (common.Hash{}) {
+		return nil, fmt.Errorf("[%#x] %w", state, errSnapshotStale)
+	}
+	l := tree.layers[tip]
+	if l == nil {
+		return nil, fmt.Errorf("triedb layer [%#x] missing", tip)
+	}
+	return l, nil
 }

triedb/pathdb/layertree_test.go

@@ -0,0 +1,885 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+package pathdb
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+)
+
+func newTestLayerTree() *layerTree {
+	db := New(rawdb.NewMemoryDatabase(), nil, false)
+	l := newDiskLayer(common.Hash{0x1}, 0, db, nil, nil, newBuffer(0, nil, nil, 0))
+	t := newLayerTree(l)
+	return t
+}
+
+func TestLayerCap(t *testing.T) {
+	var cases = []struct {
+		init     func() *layerTree
+		head     common.Hash
+		layers   int
+		base     common.Hash
+		snapshot map[common.Hash]struct{}
+	}{
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			head:   common.Hash{0x4},
+			layers: 2,
+			base:   common.Hash{0x2},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x2}: {},
+				common.Hash{0x3}: {},
+				common.Hash{0x4}: {},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C3->C4 (HEAD)
+			head:   common.Hash{0x4},
+			layers: 1,
+			base:   common.Hash{0x3},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x3}: {},
+				common.Hash{0x4}: {},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C4 (HEAD)
+			head:   common.Hash{0x4},
+			layers: 0,
+			base:   common.Hash{0x4},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x4}: {},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//     ->C2'->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			head:   common.Hash{0x4a},
+			layers: 2,
+			base:   common.Hash{0x2a},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x4a}: {},
+				common.Hash{0x3a}: {},
+				common.Hash{0x2a}: {},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//     ->C2'->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C3->C4 (HEAD)
+			head:   common.Hash{0x4a},
+			layers: 1,
+			base:   common.Hash{0x3a},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x4a}: {},
+				common.Hash{0x3a}: {},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//         ->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			//     ->C3'->C4'
+			head:   common.Hash{0x4a},
+			layers: 2,
+			base:   common.Hash{0x2},
+			snapshot: map[common.Hash]struct{}{
+				common.Hash{0x4a}: {},
+				common.Hash{0x3a}: {},
+				common.Hash{0x4b}: {},
+				common.Hash{0x3b}: {},
+				common.Hash{0x2}:  {},
+			},
+		},
+	}
+	for _, c := range cases {
+		tr := c.init()
+		if err := tr.cap(c.head, c.layers); err != nil {
+			t.Fatalf("Failed to cap the layer tree %v", err)
+		}
+		if tr.bottom().root != c.base {
+			t.Fatalf("Unexpected bottom layer tree root, want %v, got %v", c.base, tr.bottom().root)
+		}
+		if len(c.snapshot) != len(tr.layers) {
+			t.Fatalf("Unexpected layer tree size, want %v, got %v", len(c.snapshot), len(tr.layers))
+		}
+		for h := range tr.layers {
+			if _, ok := c.snapshot[h]; !ok {
+				t.Fatalf("Unexpected layer %v", h)
+			}
+		}
+	}
+}
+
+func TestBaseLayer(t *testing.T) {
+	tr := newTestLayerTree()
+
+	var cases = []struct {
+		op   func()
+		base common.Hash
+	}{
+		// Chain:
+		//   C1 (HEAD)
+		{
+			func() {},
+			common.Hash{0x1},
+		},
+		// Chain:
+		//   C1->C2->C3 (HEAD)
+		{
+			func() {
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+			},
+			common.Hash{0x1},
+		},
+		// Chain:
+		//   C3 (HEAD)
+		{
+			func() {
+				tr.cap(common.Hash{0x3}, 0)
+			},
+			common.Hash{0x3},
+		},
+		// Chain:
+		//   C4->C5->C6 (HEAD)
+		{
+			func() {
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x5}, common.Hash{0x4}, 4, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x6}, common.Hash{0x5}, 5, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.cap(common.Hash{0x6}, 2)
+			},
+			common.Hash{0x4},
+		},
+	}
+	for _, c := range cases {
+		c.op()
+		if tr.base.rootHash() != c.base {
+			t.Fatalf("Unexpected base root, want %v, got: %v", c.base, tr.base.rootHash())
+		}
+	}
+}
+
+func TestDescendant(t *testing.T) {
+	var cases = []struct {
+		init      func() *layerTree
+		snapshotA map[common.Hash]map[common.Hash]struct{}
+		op        func(tr *layerTree)
+		snapshotB map[common.Hash]map[common.Hash]struct{}
+	}{
+		{
+			// Chain:
+			//   C1->C2 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}: {},
+				},
+			},
+			// Chain:
+			//   C1->C2->C3 (HEAD)
+			op: func(tr *layerTree) {
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}: {},
+					common.Hash{0x3}: {},
+				},
+				common.Hash{0x2}: {
+					common.Hash{0x3}: {},
+				},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}: {},
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x2}: {
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x3}: {
+					common.Hash{0x4}: {},
+				},
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4}, 2)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x2}: {
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x3}: {
+					common.Hash{0x4}: {},
+				},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}: {},
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x2}: {
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x3}: {
+					common.Hash{0x4}: {},
+				},
+			},
+			// Chain:
+			//   C3->C4 (HEAD)
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4}, 1)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x3}: {
+					common.Hash{0x4}: {},
+				},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}: {},
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x2}: {
+					common.Hash{0x3}: {},
+					common.Hash{0x4}: {},
+				},
+				common.Hash{0x3}: {
+					common.Hash{0x4}: {},
+				},
+			},
+			// Chain:
+			//   C4 (HEAD)
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4}, 0)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//     ->C2'->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2a}: {},
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+					common.Hash{0x2b}: {},
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x2a}: {
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x2b}: {
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x3b}: {
+					common.Hash{0x4b}: {},
+				},
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4a}, 2)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x2a}: {
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//     ->C2'->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2a}: {},
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+					common.Hash{0x2b}: {},
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x2a}: {
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x2b}: {
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x3b}: {
+					common.Hash{0x4b}: {},
+				},
+			},
+			// Chain:
+			//   C3->C4 (HEAD)
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4a}, 1)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+			},
+		},
+		{
+			// Chain:
+			//   C1->C2->C3->C4 (HEAD)
+			//         ->C3'->C4'
+			init: func() *layerTree {
+				tr := newTestLayerTree()
+				tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3a}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x3b}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false))
+				return tr
+			},
+			snapshotA: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x1}: {
+					common.Hash{0x2}:  {},
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x2}: {
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x3b}: {
+					common.Hash{0x4b}: {},
+				},
+			},
+			// Chain:
+			//   C2->C3->C4 (HEAD)
+			//     ->C3'->C4'
+			op: func(tr *layerTree) {
+				tr.cap(common.Hash{0x4a}, 2)
+			},
+			snapshotB: map[common.Hash]map[common.Hash]struct{}{
+				common.Hash{0x2}: {
+					common.Hash{0x3a}: {},
+					common.Hash{0x4a}: {},
+					common.Hash{0x3b}: {},
+					common.Hash{0x4b}: {},
+				},
+				common.Hash{0x3a}: {
+					common.Hash{0x4a}: {},
+				},
+				common.Hash{0x3b}: {
+					common.Hash{0x4b}: {},
+				},
+			},
+		},
+	}
+	check := func(setA, setB map[common.Hash]map[common.Hash]struct{}) bool {
+		if len(setA) != len(setB) {
+			return false
+		}
+		for h, subA := range setA {
+			subB, ok := setB[h]
+			if !ok {
+				return false
+			}
+			if len(subA) != len(subB) {
+				return false
+			}
+			for hh := range subA {
+				if _, ok := subB[hh]; !ok {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	for _, c := range cases {
+		tr := c.init()
+		if !check(c.snapshotA, tr.descendants) {
+			t.Fatalf("Unexpected descendants")
+		}
+		c.op(tr)
+		if !check(c.snapshotB, tr.descendants) {
+			t.Fatalf("Unexpected descendants")
+		}
+	}
+}
+
+func TestAccountLookup(t *testing.T) {
+	// Chain:
+	//   C1->C2->C3->C4 (HEAD)
+	tr := newTestLayerTree() // base = 0x1
+	tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xa"), nil, nil, nil, false))
+	tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xb"), nil, nil, nil, false))
+	tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xa", "0xc"), nil, nil, nil, false))
+
+	var cases = []struct {
+		account common.Hash
+		state   common.Hash
+		expect  common.Hash
+	}{
+		{
+			// unknown account
+			common.HexToHash("0xd"), common.Hash{0x4}, common.Hash{0x1},
+		},
+		/*
+			lookup account from the top
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x4}, common.Hash{0x4},
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x4}, common.Hash{0x3},
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x4}, common.Hash{0x4},
+		},
+		/*
+			lookup account from the middle
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x3}, common.Hash{0x2},
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x3}, common.Hash{0x3},
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x3}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0xa"), common.Hash{0x2}, common.Hash{0x2},
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x2}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x2}, common.Hash{0x1}, // not found
+		},
+		/*
+			lookup account from the bottom
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+	}
+	for i, c := range cases {
+		l, err := tr.lookupAccount(c.account, c.state)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		if l.rootHash() != c.expect {
+			t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash())
+		}
+	}
+
+	// Chain:
+	//   C3->C4 (HEAD)
+	tr.cap(common.Hash{0x4}, 1)
+
+	cases2 := []struct {
+		account   common.Hash
+		state     common.Hash
+		expect    common.Hash
+		expectErr error
+	}{
+		{
+			// unknown account
+			common.HexToHash("0xd"), common.Hash{0x4}, common.Hash{0x3}, nil,
+		},
+		/*
+			lookup account from the top
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x4}, common.Hash{0x4}, nil,
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x4}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x4}, common.Hash{0x4}, nil,
+		},
+		/*
+			lookup account from the bottom
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x3}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x3}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x3}, common.Hash{0x3}, nil, // not found
+		},
+		/*
+			stale states
+		*/
+		{
+			common.HexToHash("0xa"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0xa"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0xb"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0xc"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+	}
+	for i, c := range cases2 {
+		l, err := tr.lookupAccount(c.account, c.state)
+		if c.expectErr != nil {
+			if !errors.Is(err, c.expectErr) {
+				t.Fatalf("%d: unexpected error, want %v, got %v", i, c.expectErr, err)
+			}
+		}
+		if c.expectErr == nil {
+			if err != nil {
+				t.Fatalf("%d: %v", i, err)
+			}
+			if l.rootHash() != c.expect {
+				t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash())
+			}
+		}
+	}
+}
+
+func TestStorageLookup(t *testing.T) {
+	// Chain:
+	//   C1->C2->C3->C4 (HEAD)
+	tr := newTestLayerTree() // base = 0x1
+	tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1"}}, nil), nil, nil, false))
+	tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x2"}}, nil), nil, nil, false))
+	tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(),
+		NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1", "0x3"}}, nil), nil, nil, false))
+
+	var cases = []struct {
+		storage common.Hash
+		state   common.Hash
+		expect  common.Hash
+	}{
+		{
+			// unknown storage slot
+			common.HexToHash("0x4"), common.Hash{0x4}, common.Hash{0x1},
+		},
+		/*
+			lookup storage slot from the top
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x4}, common.Hash{0x4},
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x4}, common.Hash{0x3},
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x4}, common.Hash{0x4},
+		},
+		/*
+			lookup storage slot from the middle
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x3}, common.Hash{0x2},
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x3}, common.Hash{0x3},
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x3}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0x1"), common.Hash{0x2}, common.Hash{0x2},
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x2}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x2}, common.Hash{0x1}, // not found
+		},
+		/*
+			lookup storage slot from the bottom
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x1}, common.Hash{0x1}, // not found
+		},
+	}
+	for i, c := range cases {
+		l, err := tr.lookupStorage(common.HexToHash("0xa"), c.storage, c.state)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		if l.rootHash() != c.expect {
+			t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash())
+		}
+	}
+
+	// Chain:
+	//   C3->C4 (HEAD)
+	tr.cap(common.Hash{0x4}, 1)
+
+	cases2 := []struct {
+		storage   common.Hash
+		state     common.Hash
+		expect    common.Hash
+		expectErr error
+	}{
+		{
+			// unknown storage slot
+			common.HexToHash("0x4"), common.Hash{0x4}, common.Hash{0x3}, nil,
+		},
+		/*
+			lookup account from the top
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x4}, common.Hash{0x4}, nil,
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x4}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x4}, common.Hash{0x4}, nil,
+		},
+		/*
+			lookup account from the bottom
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x3}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x3}, common.Hash{0x3}, nil,
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x3}, common.Hash{0x3}, nil, // not found
+		},
+		/*
+			stale states
+		*/
+		{
+			common.HexToHash("0x1"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x2}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0x1"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0x2"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+		{
+			common.HexToHash("0x3"), common.Hash{0x1}, common.Hash{}, errSnapshotStale,
+		},
+	}
+	for i, c := range cases2 {
+		l, err := tr.lookupStorage(common.HexToHash("0xa"), c.storage, c.state)
+		if c.expectErr != nil {
+			if !errors.Is(err, c.expectErr) {
+				t.Fatalf("%d: unexpected error, want %v, got %v", i, c.expectErr, err)
+			}
+		}
+		if c.expectErr == nil {
+			if err != nil {
+				t.Fatalf("%d: %v", i, err)
+			}
+			if l.rootHash() != c.expect {
+				t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash())
+			}
+		}
+	}
+}

triedb/pathdb/lookup.go

@@ -0,0 +1,278 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package pathdb
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"golang.org/x/sync/errgroup"
+)
+
+// storageKey returns a key for uniquely identifying the storage slot.
+func storageKey(accountHash common.Hash, slotHash common.Hash) [64]byte {
+	var key [64]byte
+	copy(key[:32], accountHash[:])
+	copy(key[32:], slotHash[:])
+	return key
+}
+
+// lookup is an internal structure used to efficiently determine the layer in
+// which a state entry resides.
+type lookup struct {
+	// accounts represents the mutation history for specific accounts.
+	// The key is the account address hash, and the value is a slice
+	// of **diff layer** IDs indicating where the account was modified,
+	// with the order from oldest to newest.
+	accounts map[common.Hash][]common.Hash
+
+	// storages represents the mutation history for specific storage
+	// slot. The key is the account address hash and the storage key
+	// hash, the value is a slice of **diff layer** IDs indicating
+	// where the slot was modified, with the order from oldest to newest.
+	storages map[[64]byte][]common.Hash
+
+	// descendant is the callback indicating whether the layer with
+	// given root is a descendant of the one specified by `ancestor`.
+	descendant func(state common.Hash, ancestor common.Hash) bool
+}
+
+// newLookup initializes the lookup structure.
+func newLookup(head layer, descendant func(state common.Hash, ancestor common.Hash) bool) *lookup {
+	var (
+		current = head
+		layers  []layer
+	)
+	for current != nil {
+		layers = append(layers, current)
+		current = current.parentLayer()
+	}
+	l := &lookup{
+		accounts:   make(map[common.Hash][]common.Hash),
+		storages:   make(map[[64]byte][]common.Hash),
+		descendant: descendant,
+	}
+	// Apply the diff layers from bottom to top
+	for i := len(layers) - 1; i >= 0; i-- {
+		switch diff := layers[i].(type) {
+		case *diskLayer:
+			continue
+		case *diffLayer:
+			l.addLayer(diff)
+		}
+	}
+	return l
+}
+
+// accountTip traverses the layer list associated with the given account in
+// reverse order to locate the first entry that either matches the specified
+// stateID or is a descendant of it.
+//
+// If found, the account data corresponding to the supplied stateID resides
+// in that layer. Otherwise, two scenarios are possible:
+//
+// (a) the account remains unmodified from the current disk layer up to the state
+// layer specified by the stateID: fallback to the disk layer for data retrieval,
+// (b) or the layer specified by the stateID is stale: reject the data retrieval.
+func (l *lookup) accountTip(accountHash common.Hash, stateID common.Hash, base common.Hash) common.Hash {
+	// Traverse the mutation history from latest to oldest one. Several
+	// scenarios are possible:
+	//
+	// Chain:
+	//     D->C1->C2->C3->C4 (HEAD)
+	//      ->C1'->C2'->C3'
+	// State:
+	//     x: [C1, C1', C3', C3]
+	//     y: []
+	//
+	// - (x, C4) => C3
+	// - (x, C3) => C3
+	// - (x, C2) => C1
+	// - (x, C3') => C3'
+	// - (x, C2') => C1'
+	// - (y, C4) => D
+	// - (y, C3') => D
+	// - (y, C0) => null
+	list := l.accounts[accountHash]
+	for i := len(list) - 1; i >= 0; i-- {
+		// If the current state matches the stateID, or the requested state is a
+		// descendant of it, return the current state as the most recent one
+		// containing the modified data. Otherwise, the current state may be ahead
+		// of the requested one or belong to a different branch.
+		if list[i] == stateID || l.descendant(stateID, list[i]) {
+			return list[i]
+		}
+	}
+	// No layer matching the stateID or its descendants was found. Use the
+	// current disk layer as a fallback.
+	if base == stateID || l.descendant(stateID, base) {
+		return base
+	}
+	// The layer associated with 'stateID' is not the descendant of the current
+	// disk layer, it's already stale, return nothing.
+	return common.Hash{}
+}
+
+// storageTip traverses the layer list associated with the given account and
+// slot hash in reverse order to locate the first entry that either matches
+// the specified stateID or is a descendant of it.
+//
+// If found, the storage data corresponding to the supplied stateID resides
+// in that layer. Otherwise, two scenarios are possible:
+//
+// (a) the storage slot remains unmodified from the current disk layer up to
+// the state layer specified by the stateID: fallback to the disk layer for
+// data retrieval, (b) or the layer specified by the stateID is stale: reject
+// the data retrieval.
+func (l *lookup) storageTip(accountHash common.Hash, slotHash common.Hash, stateID common.Hash, base common.Hash) common.Hash {
+	list := l.storages[storageKey(accountHash, slotHash)]
+	for i := len(list) - 1; i >= 0; i-- {
+		// If the current state matches the stateID, or the requested state is a
+		// descendant of it, return the current state as the most recent one
+		// containing the modified data. Otherwise, the current state may be ahead
+		// of the requested one or belong to a different branch.
+		if list[i] == stateID || l.descendant(stateID, list[i]) {
+			return list[i]
+		}
+	}
+	// No layer matching the stateID or its descendants was found. Use the
+	// current disk layer as a fallback.
+	if base == stateID || l.descendant(stateID, base) {
+		return base
+	}
+	// The layer associated with 'stateID' is not the descendant of the current
+	// disk layer, it's already stale, return nothing.
+	return common.Hash{}
+}
+
+// addLayer traverses the state data retained in the specified diff layer and
+// integrates it into the lookup set.
+//
+// This function assumes that all layers older than the provided one have already
+// been processed, ensuring that layers are processed strictly in a bottom-to-top
+// order.
+func (l *lookup) addLayer(diff *diffLayer) {
+	defer func(now time.Time) {
+		lookupAddLayerTimer.UpdateSince(now)
+	}(time.Now())
+
+	var (
+		wg    sync.WaitGroup
+		state = diff.rootHash()
+	)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for accountHash := range diff.states.accountData {
+			list, exists := l.accounts[accountHash]
+			if !exists {
+				list = make([]common.Hash, 0, 16) // TODO(rjl493456442) use sync pool
+			}
+			list = append(list, state)
+			l.accounts[accountHash] = list
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for accountHash, slots := range diff.states.storageData {
+			for slotHash := range slots {
+				key := storageKey(accountHash, slotHash)
+				list, exists := l.storages[key]
+				if !exists {
+					list = make([]common.Hash, 0, 16) // TODO(rjl493456442) use sync pool
+				}
+				list = append(list, state)
+				l.storages[key] = list
+			}
+		}
+	}()
+	wg.Wait()
+}
+
+// removeFromList removes the specified element from the provided list.
+// It returns a flag indicating whether the element was found and removed.
+func removeFromList(list []common.Hash, element common.Hash) (bool, []common.Hash) {
+	// Traverse the list from oldest to newest to quickly locate the element.
+	for i := 0; i < len(list); i++ {
+		if list[i] == element {
+			if i != 0 {
+				list = append(list[:i], list[i+1:]...)
+			} else {
+				// Remove the first element by shifting the slice forward.
+				// Pros: zero-copy.
+				// Cons: may retain large backing array, causing memory leaks.
+				// Mitigation: release the array if capacity exceeds threshold.
+				list = list[1:]
+				if cap(list) > 1024 {
+					list = append(make([]common.Hash, 0, len(list)), list...)
+				}
+			}
+			return true, list
+		}
+	}
+	return false, nil
+}
+
+// removeLayer traverses the state data retained in the specified diff layer and
+// unlink them from the lookup set.
+func (l *lookup) removeLayer(diff *diffLayer) error {
+	defer func(now time.Time) {
+		lookupRemoveLayerTimer.UpdateSince(now)
+	}(time.Now())
+
+	var (
+		eg    errgroup.Group
+		state = diff.rootHash()
+	)
+	eg.Go(func() error {
+		for accountHash := range diff.states.accountData {
+			found, list := removeFromList(l.accounts[accountHash], state)
+			if !found {
+				return fmt.Errorf("account lookup is not found, %x, state: %x", accountHash, state)
+			}
+			if len(list) != 0 {
+				l.accounts[accountHash] = list
+			} else {
+				delete(l.accounts, accountHash)
+			}
+		}
+		return nil
+	})
+
+	eg.Go(func() error {
+		for accountHash, slots := range diff.states.storageData {
+			for slotHash := range slots {
+				key := storageKey(accountHash, slotHash)
+				found, list := removeFromList(l.storages[key], state)
+				if !found {
+					return fmt.Errorf("storage lookup is not found, %x %x, state: %x", accountHash, slotHash, state)
+				}
+				if len(list) != 0 {
+					l.storages[key] = list
+				} else {
+					delete(l.storages, key)
+				}
+			}
+		}
+		return nil
+	})
+	return eg.Wait()
+}

triedb/pathdb/metrics.go

@@ -72,6 +72,9 @@ var (
 	historyBuildTimeMeter  = metrics.NewRegisteredResettingTimer("pathdb/history/time", nil)
 	historyDataBytesMeter  = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
 	historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
+
+	lookupAddLayerTimer    = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil)
+	lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil)
 )
 
 // Metrics in generation

triedb/pathdb/reader.go

@@ -17,6 +17,7 @@
 package pathdb
 
 import (
+	"errors"
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -50,8 +51,10 @@ func (loc *nodeLoc) string() string {
 // reader implements the database.NodeReader interface, providing the functionalities to
 // retrieve trie nodes by wrapping the internal state layer.
 type reader struct {
-	layer       layer
+	db          *Database
+	state       common.Hash
 	noHashCheck bool
+	layer       layer
 }
 
 // Node implements database.NodeReader interface, retrieving the node with specified
@@ -94,8 +97,24 @@ func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte,
 // - the returned account data is not a copy, please don't modify it
 // - no error will be returned if the requested account is not found in database
 func (r *reader) AccountRLP(hash common.Hash) ([]byte, error) {
+	l, err := r.db.tree.lookupAccount(hash, r.state)
+	if err != nil {
+		return nil, err
+	}
+	// If the located layer is stale, fall back to the slow path to retrieve
+	// the account data. This is an edge case where the located layer is the
+	// disk layer (e.g., the requested account was not changed in all the diff
+	// layers), and it becomes stale within a very short time window.
+	//
+	// This fallback mechanism is essential, because the traversal starts from
+	// the entry point layer and goes down, the staleness of the disk layer does
+	// not affect the result unless the entry point layer is also stale.
+	blob, err := l.account(hash, 0)
+	if errors.Is(err, errSnapshotStale) {
 		return r.layer.account(hash, 0)
 	}
+	return blob, err
+}
 
 // Account directly retrieves the account associated with a particular hash in
 // the slim data format. An error will be returned if the read operation exits
@@ -105,7 +124,7 @@ func (r *reader) AccountRLP(hash common.Hash) ([]byte, error) {
 // - the returned account object is safe to modify
 // - no error will be returned if the requested account is not found in database
 func (r *reader) Account(hash common.Hash) (*types.SlimAccount, error) {
-	blob, err := r.layer.account(hash, 0)
+	blob, err := r.AccountRLP(hash)
 	if err != nil {
 		return nil, err
 	}
@@ -127,8 +146,24 @@ func (r *reader) Account(hash common.Hash) (*types.SlimAccount, error) {
 // - the returned storage data is not a copy, please don't modify it
 // - no error will be returned if the requested slot is not found in database
 func (r *reader) Storage(accountHash, storageHash common.Hash) ([]byte, error) {
+	l, err := r.db.tree.lookupStorage(accountHash, storageHash, r.state)
+	if err != nil {
+		return nil, err
+	}
+	// If the located layer is stale, fall back to the slow path to retrieve
+	// the storage data. This is an edge case where the located layer is the
+	// disk layer (e.g., the requested account was not changed in all the diff
+	// layers), and it becomes stale within a very short time window.
+	//
+	// This fallback mechanism is essential, because the traversal starts from
+	// the entry point layer and goes down, the staleness of the disk layer does
+	// not affect the result unless the entry point layer is also stale.
+	blob, err := l.storage(accountHash, storageHash, 0)
+	if errors.Is(err, errSnapshotStale) {
 		return r.layer.storage(accountHash, storageHash, 0)
 	}
+	return blob, err
+}
 
 // NodeReader retrieves a layer belonging to the given state root.
 func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) {
@@ -136,7 +171,12 @@ func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) {
 	if layer == nil {
 		return nil, fmt.Errorf("state %#x is not available", root)
 	}
-	return &reader{layer: layer, noHashCheck: db.isVerkle}, nil
+	return &reader{
+		db:          db,
+		state:       root,
+		noHashCheck: db.isVerkle,
+		layer:       layer,
+	}, nil
 }
 
 // StateReader returns a reader that allows access to the state data associated
@@ -146,5 +186,9 @@ func (db *Database) StateReader(root common.Hash) (database.StateReader, error)
 	if layer == nil {
 		return nil, fmt.Errorf("state %#x is not available", root)
 	}
-	return &reader{layer: layer}, nil
+	return &reader{
+		db:    db,
+		state: root,
+		layer: layer,
+	}, nil
 }