From 92666a418909579afc3b7448890740a9ef9edf6e Mon Sep 17 00:00:00 2001 From: devopsbo3 <69951731+devopsbo3@users.noreply.github.com> Date: Fri, 10 Nov 2023 12:27:53 -0600 Subject: [PATCH] Revert "core, eth, trie: expose more detailed dirty ram tracking for diff layers (#27971)" This reverts commit 2aa9c9c87b72eb0754bd912cd083f3cc5bb62c8d. --- core/blockchain.go | 14 +++++--------- core/blockchain_insert.go | 13 ++----------- core/blockchain_test.go | 2 +- core/state/snapshot/snapshot.go | 18 ------------------ eth/state_accessor.go | 2 +- eth/tracers/api.go | 4 ++-- trie/database.go | 24 ++++++++++-------------- trie/triedb/hashdb/database.go | 7 ++----- trie/triedb/pathdb/database.go | 8 ++++---- 9 files changed, 27 insertions(+), 65 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index c579123c0e..b0ea1ac4dc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1021,7 +1021,7 @@ func (bc *BlockChain) Stop() { for !bc.triegc.Empty() { triedb.Dereference(bc.triegc.PopItem()) } - if _, nodes, _ := triedb.Size(); nodes != 0 { // all memory is contained within the nodes return for hashdb + if size, _ := triedb.Size(); size != 0 { log.Error("Dangling trie nodes after full cleanup") } } @@ -1429,8 +1429,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( - _, nodes, imgs = bc.triedb.Size() // all memory is contained within the nodes return for hashdb - limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 + nodes, imgs = bc.triedb.Size() + limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024 ) if nodes > limit || imgs > 4*1024*1024 { bc.triedb.Cap(limit - ethdb.IdealBatchSize) @@ -1866,12 +1866,8 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) stats.processed++ stats.usedGas += usedGas - var snapDiffItems, snapBufItems common.StorageSize - if bc.snaps != nil { - snapDiffItems, snapBufItems = bc.snaps.Size() - } - trieDiffNodes, trieBufNodes, _ := bc.triedb.Size() - stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead) + dirty, _ := bc.triedb.Size() + stats.report(chain, it.index, dirty, setHead) if !setHead { // After merge we expect few side chains. Simply count diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index 9bf662b6b7..8f496e182c 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second // report prints statistics if some number of blocks have been processed // or more than a few seconds have passed since the last message. -func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, snapBufItems, trieDiffNodes, triebufNodes common.StorageSize, setHead bool) { +func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize, setHead bool) { // Fetch the timings for the batch var ( now = mclock.Now() @@ -63,16 +63,7 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute { context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...) } - if snapDiffItems != 0 || snapBufItems != 0 { // snapshots enabled - context = append(context, []interface{}{"snapdiffs", snapDiffItems}...) - if snapBufItems != 0 { // future snapshot refactor - context = append(context, []interface{}{"snapdirty", snapBufItems}...) - } - } - if trieDiffNodes != 0 { // pathdb - context = append(context, []interface{}{"triediffs", trieDiffNodes}...) - } - context = append(context, []interface{}{"triedirty", triebufNodes}...) + context = append(context, []interface{}{"dirty", dirty}...) if st.queued > 0 { context = append(context, []interface{}{"queued", st.queued}...) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 992f8d168f..a365cb5ed5 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1844,7 +1844,7 @@ func TestTrieForkGC(t *testing.T) { chain.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) chain.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) } - if _, nodes, _ := chain.TrieDB().Size(); nodes > 0 { // all memory is returned in the nodes return for hashdb + if nodes, _ := chain.TrieDB().Size(); nodes > 0 { t.Fatalf("stale tries still alive after garbase collection") } } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 635e604378..efc0fc26af 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -852,21 +852,3 @@ func (t *Tree) DiskRoot() common.Hash { return t.diskRoot() } - -// Size returns the memory usage of the diff layers above the disk layer and the -// dirty nodes buffered in the disk layer. Currently, the implementation uses a -// special diff layer (the first) as an aggregator simulating a dirty buffer, so -// the second return will always be 0. However, this will be made consistent with -// the pathdb, which will require a second return. -func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) { - t.lock.RLock() - defer t.lock.RUnlock() - - var size common.StorageSize - for _, layer := range t.layers { - if layer, ok := layer.(*diffLayer); ok { - size += common.StorageSize(layer.memory) - } - } - return size, 0 -} diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 24694df66c..7418a92372 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -168,7 +168,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u parent = root } if report { - _, nodes, imgs := triedb.Size() // all memory is contained within the nodes return in hashdb + nodes, imgs := triedb.Size() log.Info("Historical state regenerated", "block", current.NumberU64(), "elapsed", time.Since(start), "nodes", nodes, "preimages", imgs) } return statedb, func() { triedb.Dereference(block.Root()) }, nil diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 8c8e09a149..740a38ab9f 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -369,8 +369,8 @@ func (api *API) traceChain(start, end *types.Block, config *TraceConfig, closed // if the relevant state is available in disk. var preferDisk bool if statedb != nil { - s1, s2, s3 := statedb.Database().TrieDB().Size() - preferDisk = s1+s2+s3 > defaultTracechainMemLimit + s1, s2 := statedb.Database().TrieDB().Size() + preferDisk = s1+s2 > defaultTracechainMemLimit } statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, statedb, false, preferDisk) if err != nil { diff --git a/trie/database.go b/trie/database.go index 42115f4886..bac3e984ce 100644 --- a/trie/database.go +++ b/trie/database.go @@ -55,12 +55,9 @@ type backend interface { // according to the state scheme. Initialized(genesisRoot common.Hash) bool - // Size returns the current storage size of the diff layers on top of the - // disk layer and the storage size of the nodes cached in the disk layer. - // - // For hash scheme, there is no differentiation between diff layer nodes - // and dirty disk layer nodes, so both are merged into the second return. - Size() (common.StorageSize, common.StorageSize) + // Size returns the current storage size of the memory cache in front of the + // persistent database layer. + Size() common.StorageSize // Update performs a state transition by committing dirty nodes contained // in the given set in order to update state from the specified parent to @@ -168,19 +165,18 @@ func (db *Database) Commit(root common.Hash, report bool) error { return db.backend.Commit(root, report) } -// Size returns the storage size of diff layer nodes above the persistent disk -// layer, the dirty nodes buffered within the disk layer, and the size of cached -// preimages. -func (db *Database) Size() (common.StorageSize, common.StorageSize, common.StorageSize) { +// Size returns the storage size of dirty trie nodes in front of the persistent +// database and the size of cached preimages. +func (db *Database) Size() (common.StorageSize, common.StorageSize) { var ( - diffs, nodes common.StorageSize - preimages common.StorageSize + storages common.StorageSize + preimages common.StorageSize ) - diffs, nodes = db.backend.Size() + storages = db.backend.Size() if db.preimages != nil { preimages = db.preimages.size() } - return diffs, nodes, preimages + return storages, preimages } // Initialized returns an indicator if the state data is already initialized diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index 764ab24ec8..b3ae54dbe3 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -624,10 +624,7 @@ func (db *Database) Update(root common.Hash, parent common.Hash, block uint64, n // Size returns the current storage size of the memory cache in front of the // persistent database layer. -// -// The first return will always be 0, representing the memory stored in unbounded -// diff layers above the dirty cache. This is only available in pathdb. -func (db *Database) Size() (common.StorageSize, common.StorageSize) { +func (db *Database) Size() common.StorageSize { db.lock.RLock() defer db.lock.RUnlock() @@ -635,7 +632,7 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // the total memory consumption, the maintenance metadata is also needed to be // counted. var metadataSize = common.StorageSize(len(db.dirties) * cachedNodeSize) - return 0, db.dirtiesSize + db.childrenSize + metadataSize + return db.dirtiesSize + db.childrenSize + metadataSize } // Close closes the trie database and releases all held resources. diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 18cc36ffc3..0382740046 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -383,16 +383,16 @@ func (db *Database) Close() error { // Size returns the current storage size of the memory cache in front of the // persistent database layer. -func (db *Database) Size() (diffs common.StorageSize, nodes common.StorageSize) { +func (db *Database) Size() (size common.StorageSize) { db.tree.forEach(func(layer layer) { if diff, ok := layer.(*diffLayer); ok { - diffs += common.StorageSize(diff.memory) + size += common.StorageSize(diff.memory) } if disk, ok := layer.(*diskLayer); ok { - nodes += disk.size() + size += disk.size() } }) - return diffs, nodes + return size } // Initialized returns an indicator if the state data is already