From 8b9f2d4e36dfcdb2af94dc0d3b63f30c8ad159a9 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 28 May 2025 19:31:42 +0800 Subject: [PATCH 01/10] triedb/pathdb: introduce lookup structure to optimize state access (#30971) This pull request introduces a mechanism to improve state lookup efficiency in pathdb by maintaining a lookup structure that eliminates unnecessary iteration over diff layers. The core idea is to track a mutation history for each dirty state entry residing in the diff layers. This history records the state roots of all layers in which the entry was modified, sorted from oldest to newest. During state lookup, this mutation history is queried to find the most recent layer whose state root either matches the target root or is a descendant of it. This allows us to quickly identify the layer containing the relevant data, avoiding the need to iterate through all diff layers from top to bottom. Besides, the overhead for state lookup is constant, no matter how many diff layers are retained in the pathdb, which unlocks the potential to hold more diff layers. Of course, maintaining this lookup structure introduces some overhead. For each state transition, we need to: (a) update the mutation records for the modified state entries, and (b) remove stale mutation records associated with outdated layers. On our benchmark machine, it will introduce around 1ms overhead which is acceptable. --- triedb/pathdb/database.go | 4 +- triedb/pathdb/difflayer.go | 4 +- triedb/pathdb/disklayer.go | 9 - triedb/pathdb/layertree.go | 199 +++++-- triedb/pathdb/layertree_test.go | 885 ++++++++++++++++++++++++++++++++ triedb/pathdb/lookup.go | 278 ++++++++++ triedb/pathdb/metrics.go | 3 + triedb/pathdb/reader.go | 56 +- 8 files changed, 1385 insertions(+), 53 deletions(-) create mode 100644 triedb/pathdb/layertree_test.go create mode 100644 triedb/pathdb/lookup.go diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index c206460315..3174a7c964 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -487,7 +487,7 @@ func (db *Database) Enable(root common.Hash) error { // Re-construct a new disk layer backed by persistent state // and schedule the state snapshot generation if it's permitted. - db.tree.reset(generateSnapshot(db, root, db.isVerkle || db.config.SnapshotNoBuild)) + db.tree.init(generateSnapshot(db, root, db.isVerkle || db.config.SnapshotNoBuild)) log.Info("Rebuilt trie database", "root", root) return nil } @@ -529,7 +529,7 @@ func (db *Database) Recover(root common.Hash) error { // reset layer with newly created disk layer. It must be // done after each revert operation, otherwise the new // disk layer won't be accessible from outside. - db.tree.reset(dl) + db.tree.init(dl) } rawdb.DeleteTrieJournal(db.diskdb) diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index 2e1d83c74a..ac05b4a0fb 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -156,7 +156,7 @@ func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes *no } // persist flushes the diff layer and all its parent layers to disk layer. -func (dl *diffLayer) persist(force bool) (layer, error) { +func (dl *diffLayer) persist(force bool) (*diskLayer, error) { if parent, ok := dl.parentLayer().(*diffLayer); ok { // Hold the lock to prevent any read operation until the new // parent is linked correctly. @@ -183,7 +183,7 @@ func (dl *diffLayer) size() uint64 { // diffToDisk merges a bottom-most diff into the persistent disk layer underneath // it. The method will panic if called onto a non-bottom-most diff layer. -func diffToDisk(layer *diffLayer, force bool) (layer, error) { +func diffToDisk(layer *diffLayer, force bool) (*diskLayer, error) { disk, ok := layer.parentLayer().(*diskLayer) if !ok { panic(fmt.Sprintf("unknown layer type: %T", layer.parentLayer())) diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index bee4cd1e8a..1c9efb024b 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -94,15 +94,6 @@ func (dl *diskLayer) setGenerator(generator *generator) { dl.generator = generator } -// isStale return whether this layer has become stale (was flattened across) or if -// it's still live. -func (dl *diskLayer) isStale() bool { - dl.lock.RLock() - defer dl.lock.RUnlock() - - return dl.stale -} - // markStale sets the stale flag as true. func (dl *diskLayer) markStale() { dl.lock.Lock() diff --git a/triedb/pathdb/layertree.go b/triedb/pathdb/layertree.go index 85a5e470e7..b2f3f7f37d 100644 --- a/triedb/pathdb/layertree.go +++ b/triedb/pathdb/layertree.go @@ -31,29 +31,50 @@ import ( // thread-safe to use. However, callers need to ensure the thread-safety // of the referenced layer by themselves. type layerTree struct { - lock sync.RWMutex + base *diskLayer layers map[common.Hash]layer + + // descendants is a two-dimensional map where the keys represent + // an ancestor state root, and the values are the state roots of + // all its descendants. + // + // For example: r -> [c1, c2, ..., cn], where c1 through cn are + // the descendants of state r. + // + // This map includes all the existing diff layers and the disk layer. + descendants map[common.Hash]map[common.Hash]struct{} + lookup *lookup + lock sync.RWMutex } // newLayerTree constructs the layerTree with the given head layer. func newLayerTree(head layer) *layerTree { tree := new(layerTree) - tree.reset(head) + tree.init(head) return tree } -// reset initializes the layerTree by the given head layer. -// All the ancestors will be iterated out and linked in the tree. -func (tree *layerTree) reset(head layer) { +// init initializes the layerTree by the given head layer. +func (tree *layerTree) init(head layer) { tree.lock.Lock() defer tree.lock.Unlock() - var layers = make(map[common.Hash]layer) - for head != nil { - layers[head.rootHash()] = head - head = head.parentLayer() + current := head + tree.layers = make(map[common.Hash]layer) + tree.descendants = make(map[common.Hash]map[common.Hash]struct{}) + + for { + tree.layers[current.rootHash()] = current + tree.fillAncestors(current) + + parent := current.parentLayer() + if parent == nil { + break + } + current = parent } - tree.layers = layers + tree.base = current.(*diskLayer) // panic if it's not a disk layer + tree.lookup = newLookup(head, tree.isDescendant) } // get retrieves a layer belonging to the given state root. @@ -64,6 +85,43 @@ func (tree *layerTree) get(root common.Hash) layer { return tree.layers[root] } +// isDescendant returns whether the specified layer with given root is a +// descendant of a specific ancestor. +// +// This function assumes the read lock has been held. +func (tree *layerTree) isDescendant(root common.Hash, ancestor common.Hash) bool { + subset := tree.descendants[ancestor] + if subset == nil { + return false + } + _, ok := subset[root] + return ok +} + +// fillAncestors identifies the ancestors of the given layer and populates the +// descendants set. The ancestors include the diff layers below the supplied +// layer and also the disk layer. +// +// This function assumes the write lock has been held. +func (tree *layerTree) fillAncestors(layer layer) { + hash := layer.rootHash() + for { + parent := layer.parentLayer() + if parent == nil { + break + } + layer = parent + + phash := parent.rootHash() + subset := tree.descendants[phash] + if subset == nil { + subset = make(map[common.Hash]struct{}) + tree.descendants[phash] = subset + } + subset[hash] = struct{}{} + } +} + // forEach iterates the stored layers inside and applies the // given callback on them. func (tree *layerTree) forEach(onLayer func(layer)) { @@ -101,8 +159,16 @@ func (tree *layerTree) add(root common.Hash, parentRoot common.Hash, block uint6 l := parent.update(root, parent.stateID()+1, block, newNodeSet(nodes.Flatten()), states) tree.lock.Lock() + defer tree.lock.Unlock() + + // Link the given layer into the layer set tree.layers[l.rootHash()] = l - tree.lock.Unlock() + + // Link the given layer into its ancestors (up to the current disk layer) + tree.fillAncestors(l) + + // Link the given layer into the state mutation history + tree.lookup.addLayer(l) return nil } @@ -127,8 +193,16 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { if err != nil { return err } - // Replace the entire layer tree with the flat base - tree.layers = map[common.Hash]layer{base.rootHash(): base} + tree.base = base + + // Reset the layer tree with the single new disk layer + tree.layers = map[common.Hash]layer{ + base.rootHash(): base, + } + // Resets the descendants map, since there's only a single disk layer + // with no descendants. + tree.descendants = make(map[common.Hash]map[common.Hash]struct{}) + tree.lookup = newLookup(base, tree.isDescendant) return nil } // Dive until we run out of layers or reach the persistent database @@ -143,6 +217,11 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { } // We're out of layers, flatten anything below, stopping if it's the disk or if // the memory limit is not yet exceeded. + var ( + err error + replaced layer + newBase *diskLayer + ) switch parent := diff.parentLayer().(type) { case *diskLayer: return nil @@ -152,14 +231,33 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { // parent is linked correctly. diff.lock.Lock() - base, err := parent.persist(false) + // Hold the reference of the original layer being replaced + replaced = parent + + // Replace the original parent layer with new disk layer. The procedure + // can be illustrated as below: + // + // Before change: + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C2'->C3'->C4' + // + // After change: + // Chain: + // (a) C3->C4 (HEAD) + // (b) C1->C2 + // ->C2'->C3'->C4' + // The original C3 is replaced by the new base (with root C3) + // Dangling layers in (b) will be removed later + newBase, err = parent.persist(false) if err != nil { diff.lock.Unlock() return err } - tree.layers[base.rootHash()] = base - diff.parent = base + tree.layers[newBase.rootHash()] = newBase + // Link the new parent and release the lock + diff.parent = newBase diff.lock.Unlock() default: @@ -173,19 +271,28 @@ func (tree *layerTree) cap(root common.Hash, layers int) error { children[parent] = append(children[parent], root) } } + clearDiff := func(layer layer) { + diff, ok := layer.(*diffLayer) + if !ok { + return + } + tree.lookup.removeLayer(diff) + } var remove func(root common.Hash) remove = func(root common.Hash) { + clearDiff(tree.layers[root]) + + // Unlink the layer from the layer tree and cascade to its children + delete(tree.descendants, root) delete(tree.layers, root) for _, child := range children[root] { remove(child) } delete(children, root) } - for root, layer := range tree.layers { - if dl, ok := layer.(*diskLayer); ok && dl.isStale() { - remove(root) - } - } + remove(tree.base.rootHash()) // remove the old/stale disk layer + clearDiff(replaced) // remove the lookup data of the stale parent being replaced + tree.base = newBase // update the base layer with newly constructed one return nil } @@ -194,17 +301,41 @@ func (tree *layerTree) bottom() *diskLayer { tree.lock.RLock() defer tree.lock.RUnlock() - if len(tree.layers) == 0 { - return nil // Shouldn't happen, empty tree - } - // pick a random one as the entry point - var current layer - for _, layer := range tree.layers { - current = layer - break - } - for current.parentLayer() != nil { - current = current.parentLayer() - } - return current.(*diskLayer) + return tree.base +} + +// lookupAccount returns the layer that is guaranteed to contain the account data +// corresponding to the specified state root being queried. +func (tree *layerTree) lookupAccount(accountHash common.Hash, state common.Hash) (layer, error) { + // Hold the read lock to prevent the unexpected layer changes + tree.lock.RLock() + defer tree.lock.RUnlock() + + tip := tree.lookup.accountTip(accountHash, state, tree.base.root) + if tip == (common.Hash{}) { + return nil, fmt.Errorf("[%#x] %w", state, errSnapshotStale) + } + l := tree.layers[tip] + if l == nil { + return nil, fmt.Errorf("triedb layer [%#x] missing", tip) + } + return l, nil +} + +// lookupStorage returns the layer that is guaranteed to contain the storage slot +// data corresponding to the specified state root being queried. +func (tree *layerTree) lookupStorage(accountHash common.Hash, slotHash common.Hash, state common.Hash) (layer, error) { + // Hold the read lock to prevent the unexpected layer changes + tree.lock.RLock() + defer tree.lock.RUnlock() + + tip := tree.lookup.storageTip(accountHash, slotHash, state, tree.base.root) + if tip == (common.Hash{}) { + return nil, fmt.Errorf("[%#x] %w", state, errSnapshotStale) + } + l := tree.layers[tip] + if l == nil { + return nil, fmt.Errorf("triedb layer [%#x] missing", tip) + } + return l, nil } diff --git a/triedb/pathdb/layertree_test.go b/triedb/pathdb/layertree_test.go new file mode 100644 index 0000000000..a72ff5899e --- /dev/null +++ b/triedb/pathdb/layertree_test.go @@ -0,0 +1,885 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import ( + "errors" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/trie/trienode" +) + +func newTestLayerTree() *layerTree { + db := New(rawdb.NewMemoryDatabase(), nil, false) + l := newDiskLayer(common.Hash{0x1}, 0, db, nil, nil, newBuffer(0, nil, nil, 0)) + t := newLayerTree(l) + return t +} + +func TestLayerCap(t *testing.T) { + var cases = []struct { + init func() *layerTree + head common.Hash + layers int + base common.Hash + snapshot map[common.Hash]struct{} + }{ + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C2->C3->C4 (HEAD) + head: common.Hash{0x4}, + layers: 2, + base: common.Hash{0x2}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x2}: {}, + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C3->C4 (HEAD) + head: common.Hash{0x4}, + layers: 1, + base: common.Hash{0x3}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C4 (HEAD) + head: common.Hash{0x4}, + layers: 0, + base: common.Hash{0x4}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x4}: {}, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C2'->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C2->C3->C4 (HEAD) + head: common.Hash{0x4a}, + layers: 2, + base: common.Hash{0x2a}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x4a}: {}, + common.Hash{0x3a}: {}, + common.Hash{0x2a}: {}, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C2'->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C3->C4 (HEAD) + head: common.Hash{0x4a}, + layers: 1, + base: common.Hash{0x3a}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x4a}: {}, + common.Hash{0x3a}: {}, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + // Chain: + // C2->C3->C4 (HEAD) + // ->C3'->C4' + head: common.Hash{0x4a}, + layers: 2, + base: common.Hash{0x2}, + snapshot: map[common.Hash]struct{}{ + common.Hash{0x4a}: {}, + common.Hash{0x3a}: {}, + common.Hash{0x4b}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x2}: {}, + }, + }, + } + for _, c := range cases { + tr := c.init() + if err := tr.cap(c.head, c.layers); err != nil { + t.Fatalf("Failed to cap the layer tree %v", err) + } + if tr.bottom().root != c.base { + t.Fatalf("Unexpected bottom layer tree root, want %v, got %v", c.base, tr.bottom().root) + } + if len(c.snapshot) != len(tr.layers) { + t.Fatalf("Unexpected layer tree size, want %v, got %v", len(c.snapshot), len(tr.layers)) + } + for h := range tr.layers { + if _, ok := c.snapshot[h]; !ok { + t.Fatalf("Unexpected layer %v", h) + } + } + } +} + +func TestBaseLayer(t *testing.T) { + tr := newTestLayerTree() + + var cases = []struct { + op func() + base common.Hash + }{ + // Chain: + // C1 (HEAD) + { + func() {}, + common.Hash{0x1}, + }, + // Chain: + // C1->C2->C3 (HEAD) + { + func() { + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + }, + common.Hash{0x1}, + }, + // Chain: + // C3 (HEAD) + { + func() { + tr.cap(common.Hash{0x3}, 0) + }, + common.Hash{0x3}, + }, + // Chain: + // C4->C5->C6 (HEAD) + { + func() { + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x5}, common.Hash{0x4}, 4, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x6}, common.Hash{0x5}, 5, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.cap(common.Hash{0x6}, 2) + }, + common.Hash{0x4}, + }, + } + for _, c := range cases { + c.op() + if tr.base.rootHash() != c.base { + t.Fatalf("Unexpected base root, want %v, got: %v", c.base, tr.base.rootHash()) + } + } +} + +func TestDescendant(t *testing.T) { + var cases = []struct { + init func() *layerTree + snapshotA map[common.Hash]map[common.Hash]struct{} + op func(tr *layerTree) + snapshotB map[common.Hash]map[common.Hash]struct{} + }{ + { + // Chain: + // C1->C2 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + }, + }, + // Chain: + // C1->C2->C3 (HEAD) + op: func(tr *layerTree) { + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + common.Hash{0x3}: {}, + }, + common.Hash{0x2}: { + common.Hash{0x3}: {}, + }, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x2}: { + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x3}: { + common.Hash{0x4}: {}, + }, + }, + // Chain: + // C2->C3->C4 (HEAD) + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4}, 2) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x2}: { + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x3}: { + common.Hash{0x4}: {}, + }, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x2}: { + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x3}: { + common.Hash{0x4}: {}, + }, + }, + // Chain: + // C3->C4 (HEAD) + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4}, 1) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x3}: { + common.Hash{0x4}: {}, + }, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x2}: { + common.Hash{0x3}: {}, + common.Hash{0x4}: {}, + }, + common.Hash{0x3}: { + common.Hash{0x4}: {}, + }, + }, + // Chain: + // C4 (HEAD) + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4}, 0) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{}, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C2'->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2a}: {}, + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + common.Hash{0x2b}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x2a}: { + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + }, + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + common.Hash{0x2b}: { + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x3b}: { + common.Hash{0x4b}: {}, + }, + }, + // Chain: + // C2->C3->C4 (HEAD) + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4a}, 2) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x2a}: { + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + }, + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C2'->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2a}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2a}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x2b}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2b}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2a}: {}, + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + common.Hash{0x2b}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x2a}: { + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + }, + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + common.Hash{0x2b}: { + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x3b}: { + common.Hash{0x4b}: {}, + }, + }, + // Chain: + // C3->C4 (HEAD) + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4a}, 1) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + }, + }, + { + // Chain: + // C1->C2->C3->C4 (HEAD) + // ->C3'->C4' + init: func() *layerTree { + tr := newTestLayerTree() + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3a}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4a}, common.Hash{0x3a}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x3b}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + tr.add(common.Hash{0x4b}, common.Hash{0x3b}, 3, trienode.NewMergedNodeSet(), NewStateSetWithOrigin(nil, nil, nil, nil, false)) + return tr + }, + snapshotA: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x1}: { + common.Hash{0x2}: {}, + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x2}: { + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + common.Hash{0x3b}: { + common.Hash{0x4b}: {}, + }, + }, + // Chain: + // C2->C3->C4 (HEAD) + // ->C3'->C4' + op: func(tr *layerTree) { + tr.cap(common.Hash{0x4a}, 2) + }, + snapshotB: map[common.Hash]map[common.Hash]struct{}{ + common.Hash{0x2}: { + common.Hash{0x3a}: {}, + common.Hash{0x4a}: {}, + common.Hash{0x3b}: {}, + common.Hash{0x4b}: {}, + }, + common.Hash{0x3a}: { + common.Hash{0x4a}: {}, + }, + common.Hash{0x3b}: { + common.Hash{0x4b}: {}, + }, + }, + }, + } + check := func(setA, setB map[common.Hash]map[common.Hash]struct{}) bool { + if len(setA) != len(setB) { + return false + } + for h, subA := range setA { + subB, ok := setB[h] + if !ok { + return false + } + if len(subA) != len(subB) { + return false + } + for hh := range subA { + if _, ok := subB[hh]; !ok { + return false + } + } + } + return true + } + for _, c := range cases { + tr := c.init() + if !check(c.snapshotA, tr.descendants) { + t.Fatalf("Unexpected descendants") + } + c.op(tr) + if !check(c.snapshotB, tr.descendants) { + t.Fatalf("Unexpected descendants") + } + } +} + +func TestAccountLookup(t *testing.T) { + // Chain: + // C1->C2->C3->C4 (HEAD) + tr := newTestLayerTree() // base = 0x1 + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xa"), nil, nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xb"), nil, nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xa", "0xc"), nil, nil, nil, false)) + + var cases = []struct { + account common.Hash + state common.Hash + expect common.Hash + }{ + { + // unknown account + common.HexToHash("0xd"), common.Hash{0x4}, common.Hash{0x1}, + }, + /* + lookup account from the top + */ + { + common.HexToHash("0xa"), common.Hash{0x4}, common.Hash{0x4}, + }, + { + common.HexToHash("0xb"), common.Hash{0x4}, common.Hash{0x3}, + }, + { + common.HexToHash("0xc"), common.Hash{0x4}, common.Hash{0x4}, + }, + /* + lookup account from the middle + */ + { + common.HexToHash("0xa"), common.Hash{0x3}, common.Hash{0x2}, + }, + { + common.HexToHash("0xb"), common.Hash{0x3}, common.Hash{0x3}, + }, + { + common.HexToHash("0xc"), common.Hash{0x3}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0xa"), common.Hash{0x2}, common.Hash{0x2}, + }, + { + common.HexToHash("0xb"), common.Hash{0x2}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0xc"), common.Hash{0x2}, common.Hash{0x1}, // not found + }, + /* + lookup account from the bottom + */ + { + common.HexToHash("0xa"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0xb"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0xc"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + } + for i, c := range cases { + l, err := tr.lookupAccount(c.account, c.state) + if err != nil { + t.Fatalf("%d: %v", i, err) + } + if l.rootHash() != c.expect { + t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash()) + } + } + + // Chain: + // C3->C4 (HEAD) + tr.cap(common.Hash{0x4}, 1) + + cases2 := []struct { + account common.Hash + state common.Hash + expect common.Hash + expectErr error + }{ + { + // unknown account + common.HexToHash("0xd"), common.Hash{0x4}, common.Hash{0x3}, nil, + }, + /* + lookup account from the top + */ + { + common.HexToHash("0xa"), common.Hash{0x4}, common.Hash{0x4}, nil, + }, + { + common.HexToHash("0xb"), common.Hash{0x4}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0xc"), common.Hash{0x4}, common.Hash{0x4}, nil, + }, + /* + lookup account from the bottom + */ + { + common.HexToHash("0xa"), common.Hash{0x3}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0xb"), common.Hash{0x3}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0xc"), common.Hash{0x3}, common.Hash{0x3}, nil, // not found + }, + /* + stale states + */ + { + common.HexToHash("0xa"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0xb"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0xc"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0xa"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0xb"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0xc"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + } + for i, c := range cases2 { + l, err := tr.lookupAccount(c.account, c.state) + if c.expectErr != nil { + if !errors.Is(err, c.expectErr) { + t.Fatalf("%d: unexpected error, want %v, got %v", i, c.expectErr, err) + } + } + if c.expectErr == nil { + if err != nil { + t.Fatalf("%d: %v", i, err) + } + if l.rootHash() != c.expect { + t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash()) + } + } + } +} + +func TestStorageLookup(t *testing.T) { + // Chain: + // C1->C2->C3->C4 (HEAD) + tr := newTestLayerTree() // base = 0x1 + tr.add(common.Hash{0x2}, common.Hash{0x1}, 1, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1"}}, nil), nil, nil, false)) + tr.add(common.Hash{0x3}, common.Hash{0x2}, 2, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x2"}}, nil), nil, nil, false)) + tr.add(common.Hash{0x4}, common.Hash{0x3}, 3, trienode.NewMergedNodeSet(), + NewStateSetWithOrigin(randomAccountSet("0xa"), randomStorageSet([]string{"0xa"}, [][]string{{"0x1", "0x3"}}, nil), nil, nil, false)) + + var cases = []struct { + storage common.Hash + state common.Hash + expect common.Hash + }{ + { + // unknown storage slot + common.HexToHash("0x4"), common.Hash{0x4}, common.Hash{0x1}, + }, + /* + lookup storage slot from the top + */ + { + common.HexToHash("0x1"), common.Hash{0x4}, common.Hash{0x4}, + }, + { + common.HexToHash("0x2"), common.Hash{0x4}, common.Hash{0x3}, + }, + { + common.HexToHash("0x3"), common.Hash{0x4}, common.Hash{0x4}, + }, + /* + lookup storage slot from the middle + */ + { + common.HexToHash("0x1"), common.Hash{0x3}, common.Hash{0x2}, + }, + { + common.HexToHash("0x2"), common.Hash{0x3}, common.Hash{0x3}, + }, + { + common.HexToHash("0x3"), common.Hash{0x3}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0x1"), common.Hash{0x2}, common.Hash{0x2}, + }, + { + common.HexToHash("0x2"), common.Hash{0x2}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0x3"), common.Hash{0x2}, common.Hash{0x1}, // not found + }, + /* + lookup storage slot from the bottom + */ + { + common.HexToHash("0x1"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0x2"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + { + common.HexToHash("0x3"), common.Hash{0x1}, common.Hash{0x1}, // not found + }, + } + for i, c := range cases { + l, err := tr.lookupStorage(common.HexToHash("0xa"), c.storage, c.state) + if err != nil { + t.Fatalf("%d: %v", i, err) + } + if l.rootHash() != c.expect { + t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash()) + } + } + + // Chain: + // C3->C4 (HEAD) + tr.cap(common.Hash{0x4}, 1) + + cases2 := []struct { + storage common.Hash + state common.Hash + expect common.Hash + expectErr error + }{ + { + // unknown storage slot + common.HexToHash("0x4"), common.Hash{0x4}, common.Hash{0x3}, nil, + }, + /* + lookup account from the top + */ + { + common.HexToHash("0x1"), common.Hash{0x4}, common.Hash{0x4}, nil, + }, + { + common.HexToHash("0x2"), common.Hash{0x4}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0x3"), common.Hash{0x4}, common.Hash{0x4}, nil, + }, + /* + lookup account from the bottom + */ + { + common.HexToHash("0x1"), common.Hash{0x3}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0x2"), common.Hash{0x3}, common.Hash{0x3}, nil, + }, + { + common.HexToHash("0x3"), common.Hash{0x3}, common.Hash{0x3}, nil, // not found + }, + /* + stale states + */ + { + common.HexToHash("0x1"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0x2"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0x3"), common.Hash{0x2}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0x1"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0x2"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + { + common.HexToHash("0x3"), common.Hash{0x1}, common.Hash{}, errSnapshotStale, + }, + } + for i, c := range cases2 { + l, err := tr.lookupStorage(common.HexToHash("0xa"), c.storage, c.state) + if c.expectErr != nil { + if !errors.Is(err, c.expectErr) { + t.Fatalf("%d: unexpected error, want %v, got %v", i, c.expectErr, err) + } + } + if c.expectErr == nil { + if err != nil { + t.Fatalf("%d: %v", i, err) + } + if l.rootHash() != c.expect { + t.Errorf("Unexpected tiphash, %d, want: %x, got: %x", i, c.expect, l.rootHash()) + } + } + } +} diff --git a/triedb/pathdb/lookup.go b/triedb/pathdb/lookup.go new file mode 100644 index 0000000000..8b092730f8 --- /dev/null +++ b/triedb/pathdb/lookup.go @@ -0,0 +1,278 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package pathdb + +import ( + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/sync/errgroup" +) + +// storageKey returns a key for uniquely identifying the storage slot. +func storageKey(accountHash common.Hash, slotHash common.Hash) [64]byte { + var key [64]byte + copy(key[:32], accountHash[:]) + copy(key[32:], slotHash[:]) + return key +} + +// lookup is an internal structure used to efficiently determine the layer in +// which a state entry resides. +type lookup struct { + // accounts represents the mutation history for specific accounts. + // The key is the account address hash, and the value is a slice + // of **diff layer** IDs indicating where the account was modified, + // with the order from oldest to newest. + accounts map[common.Hash][]common.Hash + + // storages represents the mutation history for specific storage + // slot. The key is the account address hash and the storage key + // hash, the value is a slice of **diff layer** IDs indicating + // where the slot was modified, with the order from oldest to newest. + storages map[[64]byte][]common.Hash + + // descendant is the callback indicating whether the layer with + // given root is a descendant of the one specified by `ancestor`. + descendant func(state common.Hash, ancestor common.Hash) bool +} + +// newLookup initializes the lookup structure. +func newLookup(head layer, descendant func(state common.Hash, ancestor common.Hash) bool) *lookup { + var ( + current = head + layers []layer + ) + for current != nil { + layers = append(layers, current) + current = current.parentLayer() + } + l := &lookup{ + accounts: make(map[common.Hash][]common.Hash), + storages: make(map[[64]byte][]common.Hash), + descendant: descendant, + } + // Apply the diff layers from bottom to top + for i := len(layers) - 1; i >= 0; i-- { + switch diff := layers[i].(type) { + case *diskLayer: + continue + case *diffLayer: + l.addLayer(diff) + } + } + return l +} + +// accountTip traverses the layer list associated with the given account in +// reverse order to locate the first entry that either matches the specified +// stateID or is a descendant of it. +// +// If found, the account data corresponding to the supplied stateID resides +// in that layer. Otherwise, two scenarios are possible: +// +// (a) the account remains unmodified from the current disk layer up to the state +// layer specified by the stateID: fallback to the disk layer for data retrieval, +// (b) or the layer specified by the stateID is stale: reject the data retrieval. +func (l *lookup) accountTip(accountHash common.Hash, stateID common.Hash, base common.Hash) common.Hash { + // Traverse the mutation history from latest to oldest one. Several + // scenarios are possible: + // + // Chain: + // D->C1->C2->C3->C4 (HEAD) + // ->C1'->C2'->C3' + // State: + // x: [C1, C1', C3', C3] + // y: [] + // + // - (x, C4) => C3 + // - (x, C3) => C3 + // - (x, C2) => C1 + // - (x, C3') => C3' + // - (x, C2') => C1' + // - (y, C4) => D + // - (y, C3') => D + // - (y, C0) => null + list := l.accounts[accountHash] + for i := len(list) - 1; i >= 0; i-- { + // If the current state matches the stateID, or the requested state is a + // descendant of it, return the current state as the most recent one + // containing the modified data. Otherwise, the current state may be ahead + // of the requested one or belong to a different branch. + if list[i] == stateID || l.descendant(stateID, list[i]) { + return list[i] + } + } + // No layer matching the stateID or its descendants was found. Use the + // current disk layer as a fallback. + if base == stateID || l.descendant(stateID, base) { + return base + } + // The layer associated with 'stateID' is not the descendant of the current + // disk layer, it's already stale, return nothing. + return common.Hash{} +} + +// storageTip traverses the layer list associated with the given account and +// slot hash in reverse order to locate the first entry that either matches +// the specified stateID or is a descendant of it. +// +// If found, the storage data corresponding to the supplied stateID resides +// in that layer. Otherwise, two scenarios are possible: +// +// (a) the storage slot remains unmodified from the current disk layer up to +// the state layer specified by the stateID: fallback to the disk layer for +// data retrieval, (b) or the layer specified by the stateID is stale: reject +// the data retrieval. +func (l *lookup) storageTip(accountHash common.Hash, slotHash common.Hash, stateID common.Hash, base common.Hash) common.Hash { + list := l.storages[storageKey(accountHash, slotHash)] + for i := len(list) - 1; i >= 0; i-- { + // If the current state matches the stateID, or the requested state is a + // descendant of it, return the current state as the most recent one + // containing the modified data. Otherwise, the current state may be ahead + // of the requested one or belong to a different branch. + if list[i] == stateID || l.descendant(stateID, list[i]) { + return list[i] + } + } + // No layer matching the stateID or its descendants was found. Use the + // current disk layer as a fallback. + if base == stateID || l.descendant(stateID, base) { + return base + } + // The layer associated with 'stateID' is not the descendant of the current + // disk layer, it's already stale, return nothing. + return common.Hash{} +} + +// addLayer traverses the state data retained in the specified diff layer and +// integrates it into the lookup set. +// +// This function assumes that all layers older than the provided one have already +// been processed, ensuring that layers are processed strictly in a bottom-to-top +// order. +func (l *lookup) addLayer(diff *diffLayer) { + defer func(now time.Time) { + lookupAddLayerTimer.UpdateSince(now) + }(time.Now()) + + var ( + wg sync.WaitGroup + state = diff.rootHash() + ) + wg.Add(1) + go func() { + defer wg.Done() + for accountHash := range diff.states.accountData { + list, exists := l.accounts[accountHash] + if !exists { + list = make([]common.Hash, 0, 16) // TODO(rjl493456442) use sync pool + } + list = append(list, state) + l.accounts[accountHash] = list + } + }() + + wg.Add(1) + go func() { + defer wg.Done() + for accountHash, slots := range diff.states.storageData { + for slotHash := range slots { + key := storageKey(accountHash, slotHash) + list, exists := l.storages[key] + if !exists { + list = make([]common.Hash, 0, 16) // TODO(rjl493456442) use sync pool + } + list = append(list, state) + l.storages[key] = list + } + } + }() + wg.Wait() +} + +// removeFromList removes the specified element from the provided list. +// It returns a flag indicating whether the element was found and removed. +func removeFromList(list []common.Hash, element common.Hash) (bool, []common.Hash) { + // Traverse the list from oldest to newest to quickly locate the element. + for i := 0; i < len(list); i++ { + if list[i] == element { + if i != 0 { + list = append(list[:i], list[i+1:]...) + } else { + // Remove the first element by shifting the slice forward. + // Pros: zero-copy. + // Cons: may retain large backing array, causing memory leaks. + // Mitigation: release the array if capacity exceeds threshold. + list = list[1:] + if cap(list) > 1024 { + list = append(make([]common.Hash, 0, len(list)), list...) + } + } + return true, list + } + } + return false, nil +} + +// removeLayer traverses the state data retained in the specified diff layer and +// unlink them from the lookup set. +func (l *lookup) removeLayer(diff *diffLayer) error { + defer func(now time.Time) { + lookupRemoveLayerTimer.UpdateSince(now) + }(time.Now()) + + var ( + eg errgroup.Group + state = diff.rootHash() + ) + eg.Go(func() error { + for accountHash := range diff.states.accountData { + found, list := removeFromList(l.accounts[accountHash], state) + if !found { + return fmt.Errorf("account lookup is not found, %x, state: %x", accountHash, state) + } + if len(list) != 0 { + l.accounts[accountHash] = list + } else { + delete(l.accounts, accountHash) + } + } + return nil + }) + + eg.Go(func() error { + for accountHash, slots := range diff.states.storageData { + for slotHash := range slots { + key := storageKey(accountHash, slotHash) + found, list := removeFromList(l.storages[key], state) + if !found { + return fmt.Errorf("storage lookup is not found, %x %x, state: %x", accountHash, slotHash, state) + } + if len(list) != 0 { + l.storages[key] = list + } else { + delete(l.storages, key) + } + } + } + return nil + }) + return eg.Wait() +} diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 502c34fc62..6d40c5713b 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -72,6 +72,9 @@ var ( historyBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/time", nil) historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil) historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil) + + lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) + lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) ) // Metrics in generation diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 30f75d1058..bc72db34e3 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -17,6 +17,7 @@ package pathdb import ( + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -50,8 +51,10 @@ func (loc *nodeLoc) string() string { // reader implements the database.NodeReader interface, providing the functionalities to // retrieve trie nodes by wrapping the internal state layer. type reader struct { - layer layer + db *Database + state common.Hash noHashCheck bool + layer layer } // Node implements database.NodeReader interface, retrieving the node with specified @@ -94,7 +97,23 @@ func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, // - the returned account data is not a copy, please don't modify it // - no error will be returned if the requested account is not found in database func (r *reader) AccountRLP(hash common.Hash) ([]byte, error) { - return r.layer.account(hash, 0) + l, err := r.db.tree.lookupAccount(hash, r.state) + if err != nil { + return nil, err + } + // If the located layer is stale, fall back to the slow path to retrieve + // the account data. This is an edge case where the located layer is the + // disk layer (e.g., the requested account was not changed in all the diff + // layers), and it becomes stale within a very short time window. + // + // This fallback mechanism is essential, because the traversal starts from + // the entry point layer and goes down, the staleness of the disk layer does + // not affect the result unless the entry point layer is also stale. + blob, err := l.account(hash, 0) + if errors.Is(err, errSnapshotStale) { + return r.layer.account(hash, 0) + } + return blob, err } // Account directly retrieves the account associated with a particular hash in @@ -105,7 +124,7 @@ func (r *reader) AccountRLP(hash common.Hash) ([]byte, error) { // - the returned account object is safe to modify // - no error will be returned if the requested account is not found in database func (r *reader) Account(hash common.Hash) (*types.SlimAccount, error) { - blob, err := r.layer.account(hash, 0) + blob, err := r.AccountRLP(hash) if err != nil { return nil, err } @@ -127,7 +146,23 @@ func (r *reader) Account(hash common.Hash) (*types.SlimAccount, error) { // - the returned storage data is not a copy, please don't modify it // - no error will be returned if the requested slot is not found in database func (r *reader) Storage(accountHash, storageHash common.Hash) ([]byte, error) { - return r.layer.storage(accountHash, storageHash, 0) + l, err := r.db.tree.lookupStorage(accountHash, storageHash, r.state) + if err != nil { + return nil, err + } + // If the located layer is stale, fall back to the slow path to retrieve + // the storage data. This is an edge case where the located layer is the + // disk layer (e.g., the requested account was not changed in all the diff + // layers), and it becomes stale within a very short time window. + // + // This fallback mechanism is essential, because the traversal starts from + // the entry point layer and goes down, the staleness of the disk layer does + // not affect the result unless the entry point layer is also stale. + blob, err := l.storage(accountHash, storageHash, 0) + if errors.Is(err, errSnapshotStale) { + return r.layer.storage(accountHash, storageHash, 0) + } + return blob, err } // NodeReader retrieves a layer belonging to the given state root. @@ -136,7 +171,12 @@ func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { if layer == nil { return nil, fmt.Errorf("state %#x is not available", root) } - return &reader{layer: layer, noHashCheck: db.isVerkle}, nil + return &reader{ + db: db, + state: root, + noHashCheck: db.isVerkle, + layer: layer, + }, nil } // StateReader returns a reader that allows access to the state data associated @@ -146,5 +186,9 @@ func (db *Database) StateReader(root common.Hash) (database.StateReader, error) if layer == nil { return nil, fmt.Errorf("state %#x is not available", root) } - return &reader{layer: layer}, nil + return &reader{ + db: db, + state: root, + layer: layer, + }, nil } From 1695182b55f49df20bb3046a0bbc1ca2665f551d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 17:51:47 +0200 Subject: [PATCH 02/10] .gitea: adjust cron schedule (#31915) This should make the scheduled build actually run. --- .gitea/workflows/release-cron.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-cron.yml index 3a03d0ef14..11798630dd 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-cron.yml @@ -2,7 +2,7 @@ name: Release Builds (cron) on: schedule: - cron: '0 0 * * *' + - cron: '0 16 * * *' jobs: azure-cleanup: From 0110b659c1db7612664e7fdfadbe005796992931 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 18:24:43 +0200 Subject: [PATCH 03/10] .gitea: add env to scheduled builds (#31918) --- .gitea/workflows/release-cron.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-cron.yml index 11798630dd..383c9887d6 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-cron.yml @@ -3,6 +3,7 @@ name: Release Builds (cron) on: schedule: - cron: '0 16 * * *' + workflow_dispatch: jobs: azure-cleanup: @@ -17,6 +18,8 @@ jobs: cache: false - name: Run cleanup script + env: + AZURE_BLOBSTORE_TOKEN: "{{ secrets.AZURE_BLOBSTORE_TOKEN }}" run: | go run build/ci.go purge -store gethstore/builds -days 14 @@ -37,6 +40,9 @@ jobs: apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot - name: Run ci.go + env: + DOCKER_HUB_USERNAME: "{{ secrets.DOCKER_HUB_USERNAME }}" + DOCKER_HUB_PASSWORD: "{{ secrets.DOCKER_HUB_PASSWORD }}" run: | echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " From 25f6206137bcfd260a577396e34fa3e9bdafc85e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 19:01:33 +0200 Subject: [PATCH 04/10] .gitea: port more builds from travis and fix PPA env (#31919) --- .gitea/workflows/release-cron.yml | 4 +- .gitea/workflows/release.yml | 70 ++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-cron.yml index 383c9887d6..4f586f5284 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-cron.yml @@ -41,8 +41,8 @@ jobs: - name: Run ci.go env: - DOCKER_HUB_USERNAME: "{{ secrets.DOCKER_HUB_USERNAME }}" - DOCKER_HUB_PASSWORD: "{{ secrets.DOCKER_HUB_PASSWORD }}" + PPA_SIGNING_KEY: "{{ secrets.PPA_SIGNING_KEY }}" + PPA_SSH_KEY: "{{ secrets.PPA_SSH_KEY }}" run: | echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index a91bebda7f..29154c6ef0 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -5,7 +5,7 @@ on: branches: [ master ] jobs: - linux: + linux-intel: name: Linux Build runs-on: ubuntu-latest steps: @@ -17,13 +17,68 @@ jobs: go-version: 1.24 cache: false - - name: display environment + - name: Run build (amd64) + env: + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" run: | - env + go run build/ci.go install -arch amd64 -dlgo + go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - - name: Run build + - name: Run build (386) + env: + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" run: | - go run build/ci.go install + apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib + go run build/ci.go install -arch 386 -dlgo + go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + linux-arm: + name: Linux Build (arm) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24 + cache: false + + - name: Set up cross toolchain + run: | + apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + ln -s /usr/include/asm-generic /usr/include/asm + + - name: Run build (arm64) + env: + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + run: | + go run build/ci.go install -dlgo -arch arm64 -cc arm-linux-gnueabi-gcc + go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + - name: Run build (armv5) + env: + GOARM: "5" + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + run: | + go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + - name: Run build (armv6) + env: + GOARM: "6" + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + run: | + go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + + - name: Run build (armv7) + env: + GOARM: "7" + LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + run: | + go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds docker: name: Docker Image @@ -44,5 +99,8 @@ jobs: cache: false - name: Run docker build + env: + DOCKER_HUB_USERNAME: "{{ secrets.DOCKER_HUB_USERNAME }}" + DOCKER_HUB_PASSWORD: "{{ secrets.DOCKER_HUB_PASSWORD }}" run: | - go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 + go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 -upload From e817a569fb9b84e252316a29ab491d7b76ce0ad8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 20:14:48 +0200 Subject: [PATCH 05/10] .gitea: fix secrets passing (#31920) --- .gitea/workflows/release-cron.yml | 6 +++--- .gitea/workflows/release.yml | 28 ++++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-cron.yml index 4f586f5284..735404129a 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-cron.yml @@ -19,7 +19,7 @@ jobs: - name: Run cleanup script env: - AZURE_BLOBSTORE_TOKEN: "{{ secrets.AZURE_BLOBSTORE_TOKEN }}" + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} run: | go run build/ci.go purge -store gethstore/builds -days 14 @@ -41,8 +41,8 @@ jobs: - name: Run ci.go env: - PPA_SIGNING_KEY: "{{ secrets.PPA_SIGNING_KEY }}" - PPA_SSH_KEY: "{{ secrets.PPA_SSH_KEY }}" + PPA_SIGNING_KEY: ${{ secrets.PPA_SIGNING_KEY }} + PPA_SSH_KEY: ${{ secrets.PPA_SSH_KEY }} run: | echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 29154c6ef0..7675c75e16 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -19,18 +19,18 @@ jobs: - name: Run build (amd64) env: - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | go run build/ci.go install -arch amd64 -dlgo - go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - name: Run build (386) env: - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib go run build/ci.go install -arch 386 -dlgo - go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds linux-arm: name: Linux Build (arm) @@ -51,34 +51,34 @@ jobs: - name: Run build (arm64) env: - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | go run build/ci.go install -dlgo -arch arm64 -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - name: Run build (armv5) env: GOARM: "5" - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - name: Run build (armv6) env: GOARM: "6" - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - name: Run build (armv7) env: GOARM: "7" - LINUX_SIGNING_KEY: "{{ secrets.LINUX_SIGNING_KEY }}" + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds docker: name: Docker Image @@ -100,7 +100,7 @@ jobs: - name: Run docker build env: - DOCKER_HUB_USERNAME: "{{ secrets.DOCKER_HUB_USERNAME }}" - DOCKER_HUB_PASSWORD: "{{ secrets.DOCKER_HUB_PASSWORD }}" + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} run: | go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 -upload From 31d898d586b6388ce47c1965eae8f2c7114d194f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 22:40:26 +0200 Subject: [PATCH 06/10] .gitea: fix apt update (#31921) --- .gitea/workflows/release-cron.yml | 1 + .gitea/workflows/release.yml | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-cron.yml index 735404129a..1a8eea2890 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-cron.yml @@ -37,6 +37,7 @@ jobs: - name: Install deb toolchain run: | + apt-get update apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot - name: Run ci.go diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 7675c75e16..8efd453642 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -17,20 +17,32 @@ jobs: go-version: 1.24 cache: false - - name: Run build (amd64) - env: - LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + - name: Install cross toolchain + run: | + apt-get update + apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib + + - name: Build (amd64) run: | go run build/ci.go install -arch amd64 -dlgo - go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - name: Run build (386) + - name: Create archive (amd64) env: LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} run: | - apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib + go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + rm -f build/bin/* + + - name: Build (386) + run: | go run build/ci.go install -arch 386 -dlgo + + - name: Create archive (386) + env: + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + run: | go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + rm -f build/bin/* linux-arm: name: Linux Build (arm) @@ -46,7 +58,8 @@ jobs: - name: Set up cross toolchain run: | - apt-get -yq --no-install-suggests --no-install-recommends --force-yes install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross + apt-get update + apt-get -yq --no-install-suggests --no-install-recommends install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross ln -s /usr/include/asm-generic /usr/include/asm - name: Run build (arm64) From b065aa6740e4ed109b5e3efd9abadb12b439870c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 22:57:58 +0200 Subject: [PATCH 07/10] .gitea: fix archive uploads and run PPA upload on release push (#31922) --- .gitea/workflows/release-azure-cleanup.yml | 22 ++++++ .../{release-cron.yml => release-ppa.yml} | 33 +++------ .gitea/workflows/release.yml | 71 +++++++++++++------ 3 files changed, 81 insertions(+), 45 deletions(-) create mode 100644 .gitea/workflows/release-azure-cleanup.yml rename .gitea/workflows/{release-cron.yml => release-ppa.yml} (73%) diff --git a/.gitea/workflows/release-azure-cleanup.yml b/.gitea/workflows/release-azure-cleanup.yml new file mode 100644 index 0000000000..da2400ee26 --- /dev/null +++ b/.gitea/workflows/release-azure-cleanup.yml @@ -0,0 +1,22 @@ +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +jobs: + azure-cleanup: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24 + cache: false + + - name: Run cleanup script + run: | + go run build/ci.go purge -store gethstore/builds -days 14 + env: + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} diff --git a/.gitea/workflows/release-cron.yml b/.gitea/workflows/release-ppa.yml similarity index 73% rename from .gitea/workflows/release-cron.yml rename to .gitea/workflows/release-ppa.yml index 1a8eea2890..23bffe773d 100644 --- a/.gitea/workflows/release-cron.yml +++ b/.gitea/workflows/release-ppa.yml @@ -1,30 +1,14 @@ -name: Release Builds (cron) - on: schedule: - cron: '0 16 * * *' + push: + branches: + - "release/*" workflow_dispatch: jobs: - azure-cleanup: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: 1.24 - cache: false - - - name: Run cleanup script - env: - AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} - run: | - go run build/ci.go purge -store gethstore/builds -days 14 - ppa: - name: PPA Upload (master) + name: PPA Upload runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -40,10 +24,13 @@ jobs: apt-get update apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot + - name: Add launchpad to known_hosts + run: | + echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts + - name: Run ci.go + run: | + go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " env: PPA_SIGNING_KEY: ${{ secrets.PPA_SIGNING_KEY }} PPA_SSH_KEY: ${{ secrets.PPA_SSH_KEY }} - run: | - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 8efd453642..10e0f7aa7f 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -1,8 +1,8 @@ -name: Release Builds (push) - on: push: - branches: [ master ] + branches: + - "master" + - "release/*" jobs: linux-intel: @@ -26,23 +26,25 @@ jobs: run: | go run build/ci.go install -arch amd64 -dlgo - - name: Create archive (amd64) - env: - LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + - name: Create/upload archive (amd64) run: | go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds rm -f build/bin/* + env: + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} - name: Build (386) run: | go run build/ci.go install -arch 386 -dlgo - - name: Create archive (386) - env: - LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + - name: Create/upload archive (386) run: | go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds rm -f build/bin/* + env: + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} linux-arm: name: Linux Build (arm) @@ -56,42 +58,67 @@ jobs: go-version: 1.24 cache: false - - name: Set up cross toolchain + - name: Install cross toolchain run: | apt-get update apt-get -yq --no-install-suggests --no-install-recommends install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross ln -s /usr/include/asm-generic /usr/include/asm - - name: Run build (arm64) - env: - LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + - name: Build (arm64) run: | go run build/ci.go install -dlgo -arch arm64 -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds - - name: Run build (armv5) + - name: Create/upload archive (arm64) + run: | + go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + rm -fr build/bin/* + env: + LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} + + - name: Run build (arm5) + run: | + go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc + env: + GOARM: "5" + + - name: Create/upload archive (arm5) + run: | + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds env: GOARM: "5" LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} + + - name: Run build (arm6) run: | go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + env: + GOARM: "6" - - name: Run build (armv6) + - name: Create/upload archive (arm6) + run: | + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + rm -fr build/bin/* env: GOARM: "6" LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} + + - name: Run build (arm7) run: | go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + env: + GOARM: "7" - - name: Run build (armv7) + - name: Create/upload archive (arm7) + run: | + go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + rm -fr build/bin/* env: GOARM: "7" LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }} - run: | - go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc - go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds + AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }} docker: name: Docker Image From 3524499d558ce85b51d5e803154e01b9210b3913 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 23:16:51 +0200 Subject: [PATCH 08/10] .gitea: fix arm64 build (#31923) --- .gitea/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index 10e0f7aa7f..f1637bd5ae 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -66,7 +66,7 @@ jobs: - name: Build (arm64) run: | - go run build/ci.go install -dlgo -arch arm64 -cc arm-linux-gnueabi-gcc + go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc - name: Create/upload archive (arm64) run: | From e9e4e8b05ae99c8f4e22ae2e6db4f75ce5b848b6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 28 May 2025 23:19:13 +0200 Subject: [PATCH 09/10] .gitea: run release build on tag push (#31924) --- .gitea/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index f1637bd5ae..04e724ccf4 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -2,7 +2,8 @@ on: push: branches: - "master" - - "release/*" + tags: + - "v*" jobs: linux-intel: From d821f7f297259b2ed5ab0d3c99c0e4aca9663ac8 Mon Sep 17 00:00:00 2001 From: Arya Nair Date: Thu, 29 May 2025 09:50:39 +0530 Subject: [PATCH 10/10] cmd/geth, cmd/utils: log prefunded account/key in ephemeral development mode (#31898) This PR modifies the disclaimer/banner that is printed when starting up Geth in dev mode: * if the client is spun up in ephemeral dev mode with a keystore override, the address of the first (prefunded) account is printed. * if the client is spun up in ephemeral mode without a keystore override, the genesis allocation contains a single static prefunded EOA account. It's address and private key are logged. * the banner is printed at the end of client initialization to make it more prominent. Previously, it was logged towards the beginning of client initialization and subsequent logging from startup filled the terminal, pushing it out of view of the user. Other change is that we now use a static prefunded dev account instead of generating a random one when instantiating a new dev mode chain. This is an example of what the banner looks like: ``` WARN [05-28|23:05:16.475] You are running Geth in --dev mode. Please note the following: WARN [05-28|23:05:16.475] WARN [05-28|23:05:16.475] 1. This mode is only intended for fast, iterative development without assumptions on WARN [05-28|23:05:16.475] security or persistence. WARN [05-28|23:05:16.475] 2. The database is created in memory unless specified otherwise. Therefore, shutting down WARN [05-28|23:05:16.475] your computer or losing power will wipe your entire block data and chain state for WARN [05-28|23:05:16.475] your dev environment. WARN [05-28|23:05:16.475] 3. A random, pre-allocated developer account will be available and unlocked as WARN [05-28|23:05:16.475] eth.coinbase, which can be used for testing. The random dev account is temporary, WARN [05-28|23:05:16.475] stored on a ramdisk, and will be lost if your machine is restarted. WARN [05-28|23:05:16.475] 4. Mining is enabled by default. However, the client will only seal blocks if transactions WARN [05-28|23:05:16.475] are pending in the mempool. The miner's minimum accepted gas price is 1. WARN [05-28|23:05:16.475] 5. Networking is disabled; there is no listen-address, the maximum number of peers is set WARN [05-28|23:05:16.475] to 0, and discovery is disabled. WARN [05-28|23:05:16.475] WARN [05-28|23:05:16.475] WARN [05-28|23:05:16.475] Running in ephemeral mode. The following account has been prefunded in the genesis: WARN [05-28|23:05:16.475] WARN [05-28|23:05:16.475] Account WARN [05-28|23:05:16.475] ------------------ WARN [05-28|23:05:16.475] 0x71562b71999873db5b286df957af199ec94617f7 (10^49 ETH) WARN [05-28|23:05:16.475] WARN [05-28|23:05:16.475] Private Key WARN [05-28|23:05:16.475] ------------------ WARN [05-28|23:05:16.475] 0xb71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291 WARN [05-28|23:05:16.475] ``` closes #31796 --------- Co-authored-by: jwasinger --- cmd/geth/config.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ cmd/geth/main.go | 18 ------------------ cmd/utils/flags.go | 20 +++++++++++++++----- 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 4215403234..3eb2b6dd37 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/flags" @@ -180,6 +181,45 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { return stack, cfg } +// constructs the disclaimer text block which will be printed in the logs upon +// startup when Geth is running in dev mode. +func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string { + devModeBanner := `You are running Geth in --dev mode. Please note the following: + + 1. This mode is only intended for fast, iterative development without assumptions on + security or persistence. + 2. The database is created in memory unless specified otherwise. Therefore, shutting down + your computer or losing power will wipe your entire block data and chain state for + your dev environment. + 3. A random, pre-allocated developer account will be available and unlocked as + eth.coinbase, which can be used for testing. The random dev account is temporary, + stored on a ramdisk, and will be lost if your machine is restarted. + 4. Mining is enabled by default. However, the client will only seal blocks if transactions + are pending in the mempool. The miner's minimum accepted gas price is 1. + 5. Networking is disabled; there is no listen-address, the maximum number of peers is set + to 0, and discovery is disabled. +` + if !ctx.IsSet(utils.DataDirFlag.Name) { + devModeBanner += fmt.Sprintf(` + + Running in ephemeral mode. The following account has been prefunded in the genesis: + + Account + ------------------ + 0x%x (10^49 ETH) +`, cfg.Eth.Miner.PendingFeeRecipient) + if cfg.Eth.Miner.PendingFeeRecipient == utils.DeveloperAddr { + devModeBanner += fmt.Sprintf(` + Private Key + ------------------ + 0x%x +`, crypto.FromECDSA(utils.DeveloperKey)) + } + } + + return devModeBanner +} + // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) *node.Node { stack, cfg := makeConfigNode(ctx) @@ -239,6 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node { } catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) stack.RegisterLifecycle(simBeacon) + + banner := constructDevModeBanner(ctx, cfg) + for _, line := range strings.Split(banner, "\n") { + log.Warn(line) + } } else if ctx.IsSet(utils.BeaconApiFlag.Name) { // Start blsync mode. srv := rpc.NewServer() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 733c9cf230..289af90828 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -294,24 +294,6 @@ func prepare(ctx *cli.Context) { case ctx.IsSet(utils.HoodiFlag.Name): log.Info("Starting Geth on Hoodi testnet...") - case ctx.IsSet(utils.DeveloperFlag.Name): - log.Info("Starting Geth in ephemeral dev mode...") - log.Warn(`You are running Geth in --dev mode. Please note the following: - - 1. This mode is only intended for fast, iterative development without assumptions on - security or persistence. - 2. The database is created in memory unless specified otherwise. Therefore, shutting down - your computer or losing power will wipe your entire block data and chain state for - your dev environment. - 3. A random, pre-allocated developer account will be available and unlocked as - eth.coinbase, which can be used for testing. The random dev account is temporary, - stored on a ramdisk, and will be lost if your machine is restarted. - 4. Mining is enabled by default. However, the client will only seal blocks if transactions - are pending in the mempool. The miner's minimum accepted gas price is 1. - 5. Networking is disabled; there is no listen-address, the maximum number of peers is set - to 0, and discovery is disabled. -`) - case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 81c4172a53..60a2cc13f1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -984,6 +984,12 @@ var ( } ) +// default account to prefund when running Geth in dev mode +var ( + DeveloperKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + DeveloperAddr = crypto.PubkeyToAddress(DeveloperKey.PublicKey) +) + // MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // then a subdirectory of the specified datadir will be used. @@ -1769,9 +1775,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } else if accs := ks.Accounts(); len(accs) > 0 { developer = ks.Accounts()[0] } else { - developer, err = ks.NewAccount(passphrase) + developer, err = ks.ImportECDSA(DeveloperKey, passphrase) if err != nil { - Fatalf("Failed to create developer account: %v", err) + Fatalf("Failed to import developer account: %v", err) } } // Make sure the address is configured as fee recipient, otherwise @@ -1786,14 +1792,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } log.Info("Using developer account", "address", developer.Address) - // Create a new developer genesis block or reuse existing one + // configure default developer genesis which will be used unless a + // datadir is specified and a chain is preexisting at that location. cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), &developer.Address) + + // If a datadir is specified, ensure that any preexisting chain in that location + // has a configuration that is compatible with dev mode: it must be merged at genesis. if ctx.IsSet(DataDirFlag.Name) { chaindb := tryMakeReadOnlyDatabase(ctx, stack) if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) { - cfg.Genesis = nil // fallback to db content + // signal fallback to preexisting chain on disk + cfg.Genesis = nil - //validate genesis has PoS enabled in block 0 genesis, err := core.ReadGenesis(chaindb) if err != nil { Fatalf("Could not read genesis from database: %v", err)