From 225a826e0e8e3067fa5839e9f30e1306aa018327 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 17 Oct 2022 18:00:30 +0530 Subject: [PATCH 1/5] add : TriesInMemory flag --- builder/files/config.toml | 1 + core/blockchain.go | 13 ++++--- core/blockchain_test.go | 72 +++++++++++++++++------------------ internal/cli/server/config.go | 4 ++ internal/cli/server/flags.go | 7 ++++ les/handler_test.go | 4 +- les/peer.go | 2 +- les/server_requests.go | 4 +- les/test_helper.go | 2 +- 9 files changed, 61 insertions(+), 48 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index 0d01f85d71..13bf82bf93 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -122,6 +122,7 @@ syncmode = "full" # noprefetch = false # preimages = false # txlookuplimit = 2350000 + # triesinmemory = 128 [accounts] # allow-insecure-unlock = true diff --git a/core/blockchain.go b/core/blockchain.go index 48d5e966db..d49bd5f937 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -92,7 +92,6 @@ const ( txLookupCacheLimit = 1024 maxFutureBlocks = 256 maxTimeFutureBlocks = 30 - TriesInMemory = 128 // BlockChainVersion ensures that an incompatible database forces a resync from scratch. // @@ -132,6 +131,7 @@ type CacheConfig struct { TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk SnapshotLimit int // Memory allowance (MB) to use for caching snapshot entries in memory Preimages bool // Whether to store preimage of trie key to the disk + TriesInMemory uint64 // Number of recent tries to keep in memory SnapshotWait bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it } @@ -144,6 +144,7 @@ var DefaultCacheConfig = &CacheConfig{ TrieTimeLimit: 5 * time.Minute, SnapshotLimit: 256, SnapshotWait: true, + TriesInMemory: 128, } // BlockChain represents the canonical chain given a database with a genesis @@ -829,7 +830,7 @@ func (bc *BlockChain) Stop() { if !bc.cacheConfig.TrieDirtyDisabled { triedb := bc.stateCache.TrieDB() - for _, offset := range []uint64{0, 1, TriesInMemory - 1} { + for _, offset := range []uint64{0, 1, DefaultCacheConfig.TriesInMemory - 1} { if number := bc.CurrentBlock().NumberU64(); number > offset { recent := bc.GetBlockByNumber(number - offset) @@ -1297,7 +1298,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - if current := block.NumberU64(); current > TriesInMemory { + if current := block.NumberU64(); current > DefaultCacheConfig.TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() @@ -1307,7 +1308,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - chosen := current - TriesInMemory + chosen := current - DefaultCacheConfig.TriesInMemory // If we exceeded out time allowance, flush an entire trie to disk if bc.gcproc > bc.cacheConfig.TrieTimeLimit { @@ -1319,8 +1320,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } else { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/TriesInMemory) + if chosen < lastWrite+DefaultCacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", DefaultCacheConfig.TriesInMemory, "optimum", float64(chosen-lastWrite)/float64((DefaultCacheConfig.TriesInMemory))) } // Flush an entire trie and restart the counters triedb.Commit(header.Root, true, nil) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index b2c14f81ba..4d05d11198 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1652,7 +1652,7 @@ func TestTrieForkGC(t *testing.T) { db := rawdb.NewMemoryDatabase() genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*int(DefaultCacheConfig.TriesInMemory), func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) // Generate a bunch of fork blocks, each side forking from the canonical chain forks := make([]*types.Block, len(blocks)) @@ -1681,7 +1681,7 @@ func TestTrieForkGC(t *testing.T) { } } // Dereference all the recent tries and ensure no past trie is left in - for i := 0; i < TriesInMemory; i++ { + for i := 0; i < int(DefaultCacheConfig.TriesInMemory); i++ { chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) } @@ -1700,8 +1700,8 @@ func TestLargeReorgTrieGC(t *testing.T) { genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) shared, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 64, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) - original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*TriesInMemory, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) - competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) + original, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*int(DefaultCacheConfig.TriesInMemory), func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) + competitor, _ := GenerateChain(params.TestChainConfig, shared[len(shared)-1], engine, db, 2*int(DefaultCacheConfig.TriesInMemory)+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) }) // Import the shared chain and the original canonical one diskdb := rawdb.NewMemoryDatabase() @@ -1736,7 +1736,7 @@ func TestLargeReorgTrieGC(t *testing.T) { if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } - for i, block := range competitor[:len(competitor)-TriesInMemory] { + for i, block := range competitor[:len(competitor)-int(DefaultCacheConfig.TriesInMemory)] { if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: competing chain state missing", i) } @@ -1882,8 +1882,8 @@ func TestInsertReceiptChainRollback(t *testing.T) { // overtake the 'canon' chain until after it's passed canon by about 200 blocks. // // Details at: -// - https://github.com/ethereum/go-ethereum/issues/18977 -// - https://github.com/ethereum/go-ethereum/pull/18988 +// - https://github.com/ethereum/go-ethereum/issues/18977 +// - https://github.com/ethereum/go-ethereum/pull/18988 func TestLowDiffLongChain(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() @@ -1892,7 +1892,7 @@ func TestLowDiffLongChain(t *testing.T) { // We must use a pretty long chain to ensure that the fork doesn't overtake us // until after at least 128 blocks post tip - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*TriesInMemory, func(i int, b *BlockGen) { + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 6*int(DefaultCacheConfig.TriesInMemory), func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) b.OffsetTime(-9) }) @@ -1910,7 +1910,7 @@ func TestLowDiffLongChain(t *testing.T) { } // Generate fork chain, starting from an early block parent := blocks[10] - fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 8*int(DefaultCacheConfig.TriesInMemory), func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) @@ -1979,7 +1979,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) } - blocks, _ := GenerateChain(&chainConfig, genesis, genEngine, db, 2*TriesInMemory, func(i int, gen *BlockGen) { + blocks, _ := GenerateChain(&chainConfig, genesis, genEngine, db, 2*int(DefaultCacheConfig.TriesInMemory), func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, big.NewInt(int64(i+1)*params.GWei), nil), signer, key) if err != nil { t.Fatalf("failed to create tx: %v", err) @@ -1991,9 +1991,9 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - lastPrunedIndex := len(blocks) - TriesInMemory - 1 + lastPrunedIndex := len(blocks) - int(DefaultCacheConfig.TriesInMemory) - 1 lastPrunedBlock := blocks[lastPrunedIndex] - firstNonPrunedBlock := blocks[len(blocks)-TriesInMemory] + firstNonPrunedBlock := blocks[len(blocks)-int(DefaultCacheConfig.TriesInMemory)] // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { @@ -2019,7 +2019,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Generate fork chain, make it longer than canon parentIndex := lastPrunedIndex + blocksBetweenCommonAncestorAndPruneblock parent := blocks[parentIndex] - fork, _ := GenerateChain(&chainConfig, parent, genEngine, db, 2*TriesInMemory, func(i int, b *BlockGen) { + fork, _ := GenerateChain(&chainConfig, parent, genEngine, db, 2*int(DefaultCacheConfig.TriesInMemory), func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) }) // Prepend the parent(s) @@ -2046,7 +2046,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // That is: the sidechain for import contains some blocks already present in canon chain. // So the blocks are // [ Cn, Cn+1, Cc, Sn+3 ... Sm] -// ^ ^ ^ pruned +// +// ^ ^ ^ pruned func TestPrunedImportSide(t *testing.T) { //glogger := log.NewGlogHandler(log.StreamHandler(os.Stdout, log.TerminalFormat(false))) //glogger.Verbosity(3) @@ -2841,9 +2842,9 @@ func BenchmarkBlockChain_1x1000Executions(b *testing.B) { // This internally leads to a sidechain import, since the blocks trigger an // ErrPrunedAncestor error. // This may e.g. happen if -// 1. Downloader rollbacks a batch of inserted blocks and exits -// 2. Downloader starts to sync again -// 3. The blocks fetched are all known and canonical blocks +// 1. Downloader rollbacks a batch of inserted blocks and exits +// 2. Downloader starts to sync again +// 3. The blocks fetched are all known and canonical blocks func TestSideImportPrunedBlocks(t *testing.T) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() @@ -2851,7 +2852,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { genesis := (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) // Generate and import the canonical chain - blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*TriesInMemory, nil) + blocks, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 2*int(DefaultCacheConfig.TriesInMemory), nil) diskdb := rawdb.NewMemoryDatabase() (&Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(diskdb) @@ -2863,14 +2864,14 @@ func TestSideImportPrunedBlocks(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - lastPrunedIndex := len(blocks) - TriesInMemory - 1 + lastPrunedIndex := len(blocks) - int(DefaultCacheConfig.TriesInMemory) - 1 lastPrunedBlock := blocks[lastPrunedIndex] // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } - firstNonPrunedBlock := blocks[len(blocks)-TriesInMemory] + firstNonPrunedBlock := blocks[len(blocks)-int(DefaultCacheConfig.TriesInMemory)] // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) @@ -3356,20 +3357,19 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) { // TestInitThenFailCreateContract tests a pretty notorious case that happened // on mainnet over blocks 7338108, 7338110 and 7338115. -// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated -// with 0.001 ether (thus created but no code) -// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on -// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the -// deployment fails due to OOG during initcode execution -// - Block 7338115: another tx checks the balance of -// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as -// zero. +// - Block 7338108: address e771789f5cccac282f23bb7add5690e1f6ca467c is initiated +// with 0.001 ether (thus created but no code) +// - Block 7338110: a CREATE2 is attempted. The CREATE2 would deploy code on +// the same address e771789f5cccac282f23bb7add5690e1f6ca467c. However, the +// deployment fails due to OOG during initcode execution +// - Block 7338115: another tx checks the balance of +// e771789f5cccac282f23bb7add5690e1f6ca467c, and the snapshotter returned it as +// zero. // // The problem being that the snapshotter maintains a destructset, and adds items // to the destructset in case something is created "onto" an existing item. // We need to either roll back the snapDestructs, or not place it into snapDestructs // in the first place. -// func TestInitThenFailCreateContract(t *testing.T) { var ( // Generate a canonical chain to act as the main dataset @@ -3558,13 +3558,13 @@ func TestEIP2718Transition(t *testing.T) { // TestEIP1559Transition tests the following: // -// 1. A transaction whose gasFeeCap is greater than the baseFee is valid. -// 2. Gas accounting for access lists on EIP-1559 transactions is correct. -// 3. Only the transaction's tip will be received by the coinbase. -// 4. The transaction sender pays for both the tip and baseFee. -// 5. The coinbase receives only the partially realized tip when -// gasFeeCap - gasTipCap < baseFee. -// 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). +// 1. A transaction whose gasFeeCap is greater than the baseFee is valid. +// 2. Gas accounting for access lists on EIP-1559 transactions is correct. +// 3. Only the transaction's tip will be received by the coinbase. +// 4. The transaction sender pays for both the tip and baseFee. +// 5. The coinbase receives only the partially realized tip when +// gasFeeCap - gasTipCap < baseFee. +// 6. Legacy transaction behave as expected (e.g. gasPrice = gasFeeCap = gasTipCap). func TestEIP1559Transition(t *testing.T) { var ( aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index e5e3a8b03e..8bc4c3e753 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -364,6 +364,9 @@ type CacheConfig struct { // TxLookupLimit sets the maximum number of blocks from head whose tx indices are reserved. TxLookupLimit uint64 `hcl:"txlookuplimit,optional" toml:"txlookuplimit,optional"` + + // Number of block states to keep in memory (default = 128) + TriesInMemory uint64 `hcl:"triesinmemory,optional" toml:"triesinmemory,optional"` } type AccountsConfig struct { @@ -509,6 +512,7 @@ func DefaultConfig() *Config { NoPrefetch: false, Preimages: false, TxLookupLimit: 2350000, + TriesInMemory: 128, }, Accounts: &AccountsConfig{ Unlock: []string{}, diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 6e93a94de0..025ce8c80c 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -304,6 +304,13 @@ func (c *Command) Flags() *flagset.Flagset { Default: c.cliConfig.Cache.Preimages, Group: "Cache", }) + f.Uint64Flag(&flagset.Uint64Flag{ + Name: "cache.triesinmemory", + Usage: "Number of block states (tries) to keep in memory (default = 128)", + Value: &c.cliConfig.Cache.TriesInMemory, + Default: c.cliConfig.Cache.TriesInMemory, + Group: "Cache", + }) f.Uint64Flag(&flagset.Uint64Flag{ Name: "txlookuplimit", Usage: "Number of recent blocks to maintain transactions index for (default = about 56 days, 0 = entire chain)", diff --git a/les/handler_test.go b/les/handler_test.go index aba45764b3..3ceabdf8ec 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -316,7 +316,7 @@ func TestGetStaleCodeLes4(t *testing.T) { testGetStaleCode(t, 4) } func testGetStaleCode(t *testing.T, protocol int) { netconfig := testnetConfig{ - blocks: core.TriesInMemory + 4, + blocks: int(core.DefaultCacheConfig.TriesInMemory) + 4, protocol: protocol, nopruning: true, } @@ -430,7 +430,7 @@ func TestGetStaleProofLes4(t *testing.T) { testGetStaleProof(t, 4) } func testGetStaleProof(t *testing.T, protocol int) { netconfig := testnetConfig{ - blocks: core.TriesInMemory + 4, + blocks: int(core.DefaultCacheConfig.TriesInMemory) + 4, protocol: protocol, nopruning: true, } diff --git a/les/peer.go b/les/peer.go index 499429739d..a525336f0a 100644 --- a/les/peer.go +++ b/les/peer.go @@ -1058,7 +1058,7 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge // If local ethereum node is running in archive mode, advertise ourselves we have // all version state data. Otherwise only recent state is available. - stateRecent := uint64(core.TriesInMemory - blockSafetyMargin) + stateRecent := uint64(core.DefaultCacheConfig.TriesInMemory - blockSafetyMargin) if server.archiveMode { stateRecent = 0 } diff --git a/les/server_requests.go b/les/server_requests.go index bab5f733d5..3595a6ab38 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -297,7 +297,7 @@ func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) { // Refuse to search stale state data in the database since looking for // a non-exist key is kind of expensive. local := bc.CurrentHeader().Number.Uint64() - if !backend.ArchiveMode() && header.Number.Uint64()+core.TriesInMemory <= local { + if !backend.ArchiveMode() && header.Number.Uint64()+core.DefaultCacheConfig.TriesInMemory <= local { p.Log().Debug("Reject stale code request", "number", header.Number.Uint64(), "head", local) p.bumpInvalid() continue @@ -396,7 +396,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { // Refuse to search stale state data in the database since looking for // a non-exist key is kind of expensive. local := bc.CurrentHeader().Number.Uint64() - if !backend.ArchiveMode() && header.Number.Uint64()+core.TriesInMemory <= local { + if !backend.ArchiveMode() && header.Number.Uint64()+core.DefaultCacheConfig.TriesInMemory <= local { p.Log().Debug("Reject stale trie request", "number", header.Number.Uint64(), "head", local) p.bumpInvalid() continue diff --git a/les/test_helper.go b/les/test_helper.go index a099458353..d68370d508 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -377,7 +377,7 @@ func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Ha sendList = sendList.add("serveHeaders", nil) sendList = sendList.add("serveChainSince", uint64(0)) sendList = sendList.add("serveStateSince", uint64(0)) - sendList = sendList.add("serveRecentState", uint64(core.TriesInMemory-4)) + sendList = sendList.add("serveRecentState", uint64(core.DefaultCacheConfig.TriesInMemory-4)) sendList = sendList.add("txRelay", nil) sendList = sendList.add("flowControl/BL", testBufLimit) sendList = sendList.add("flowControl/MRR", testBufRecharge) From 3db339b6fe728fa47adfeef89fb6dc64ff52d495 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 17 Oct 2022 18:07:32 +0530 Subject: [PATCH 2/5] add : lint --- core/blockchain_test.go | 3 +++ les/peer.go | 2 +- les/test_helper.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 4d05d11198..d44d563b20 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1736,6 +1736,7 @@ func TestLargeReorgTrieGC(t *testing.T) { if _, err := chain.InsertChain(competitor[len(competitor)-2:]); err != nil { t.Fatalf("failed to finalize competitor chain: %v", err) } + for i, block := range competitor[:len(competitor)-int(DefaultCacheConfig.TriesInMemory)] { if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: competing chain state missing", i) @@ -1979,6 +1980,7 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon // Set the terminal total difficulty in the config gspec.Config.TerminalTotalDifficulty = big.NewInt(0) } + blocks, _ := GenerateChain(&chainConfig, genesis, genEngine, db, 2*int(DefaultCacheConfig.TriesInMemory), func(i int, gen *BlockGen) { tx, err := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("deadbeef"), big.NewInt(100), 21000, big.NewInt(int64(i+1)*params.GWei), nil), signer, key) if err != nil { @@ -2871,6 +2873,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } + firstNonPrunedBlock := blocks[len(blocks)-int(DefaultCacheConfig.TriesInMemory)] // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { diff --git a/les/peer.go b/les/peer.go index a525336f0a..46a88cfff7 100644 --- a/les/peer.go +++ b/les/peer.go @@ -1058,7 +1058,7 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge // If local ethereum node is running in archive mode, advertise ourselves we have // all version state data. Otherwise only recent state is available. - stateRecent := uint64(core.DefaultCacheConfig.TriesInMemory - blockSafetyMargin) + stateRecent := core.DefaultCacheConfig.TriesInMemory - blockSafetyMargin if server.archiveMode { stateRecent = 0 } diff --git a/les/test_helper.go b/les/test_helper.go index d68370d508..1e6fb6653f 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -377,7 +377,7 @@ func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Ha sendList = sendList.add("serveHeaders", nil) sendList = sendList.add("serveChainSince", uint64(0)) sendList = sendList.add("serveStateSince", uint64(0)) - sendList = sendList.add("serveRecentState", uint64(core.DefaultCacheConfig.TriesInMemory-4)) + sendList = sendList.add("serveRecentState", core.DefaultCacheConfig.TriesInMemory-4) sendList = sendList.add("txRelay", nil) sendList = sendList.add("flowControl/BL", testBufLimit) sendList = sendList.add("flowControl/MRR", testBufRecharge) From ec57aabf9e64d6c3f6754aa9b3ed3d483e165aac Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Tue, 18 Oct 2022 03:56:55 +0530 Subject: [PATCH 3/5] chg : minor fix --- core/blockchain.go | 13 ++++++++----- core/blockchain_test.go | 12 ++++++------ core/tests/blockchain_sethead_test.go | 2 ++ eth/backend.go | 1 + eth/ethconfig/config.go | 1 + 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d49bd5f937..5349ea8866 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -230,6 +230,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par if cacheConfig == nil { cacheConfig = DefaultCacheConfig } + if cacheConfig.TriesInMemory <= 0 { + cacheConfig.TriesInMemory = 128 + } bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit) receiptsCache, _ := lru.New(receiptsCacheLimit) @@ -830,7 +833,7 @@ func (bc *BlockChain) Stop() { if !bc.cacheConfig.TrieDirtyDisabled { triedb := bc.stateCache.TrieDB() - for _, offset := range []uint64{0, 1, DefaultCacheConfig.TriesInMemory - 1} { + for _, offset := range []uint64{0, 1, bc.cacheConfig.TriesInMemory - 1} { if number := bc.CurrentBlock().NumberU64(); number > offset { recent := bc.GetBlockByNumber(number - offset) @@ -1298,7 +1301,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive bc.triegc.Push(root, -int64(block.NumberU64())) - if current := block.NumberU64(); current > DefaultCacheConfig.TriesInMemory { + if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory { // If we exceeded our memory allowance, flush matured singleton nodes to disk var ( nodes, imgs = triedb.Size() @@ -1308,7 +1311,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. triedb.Cap(limit - ethdb.IdealBatchSize) } // Find the next state trie we need to commit - chosen := current - DefaultCacheConfig.TriesInMemory + chosen := current - bc.cacheConfig.TriesInMemory // If we exceeded out time allowance, flush an entire trie to disk if bc.gcproc > bc.cacheConfig.TrieTimeLimit { @@ -1320,8 +1323,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } else { // If we're exceeding limits but haven't reached a large enough memory gap, // warn the user that the system is becoming unstable. - if chosen < lastWrite+DefaultCacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { - log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", DefaultCacheConfig.TriesInMemory, "optimum", float64(chosen-lastWrite)/float64((DefaultCacheConfig.TriesInMemory))) + if chosen < lastWrite+bc.cacheConfig.TriesInMemory && bc.gcproc >= 2*bc.cacheConfig.TrieTimeLimit { + log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TriesInMemory, "optimum", float64(chosen-lastWrite)/float64((bc.cacheConfig.TriesInMemory))) } // Flush an entire trie and restart the counters triedb.Commit(header.Root, true, nil) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index d44d563b20..fa6b61225e 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1681,7 +1681,7 @@ func TestTrieForkGC(t *testing.T) { } } // Dereference all the recent tries and ensure no past trie is left in - for i := 0; i < int(DefaultCacheConfig.TriesInMemory); i++ { + for i := 0; i < int(chain.cacheConfig.TriesInMemory); i++ { chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root()) chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root()) } @@ -1737,7 +1737,7 @@ func TestLargeReorgTrieGC(t *testing.T) { t.Fatalf("failed to finalize competitor chain: %v", err) } - for i, block := range competitor[:len(competitor)-int(DefaultCacheConfig.TriesInMemory)] { + for i, block := range competitor[:len(competitor)-int(chain.cacheConfig.TriesInMemory)] { if node, _ := chain.stateCache.TrieDB().Node(block.Root()); node != nil { t.Fatalf("competitor %d: competing chain state missing", i) } @@ -1993,9 +1993,9 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - lastPrunedIndex := len(blocks) - int(DefaultCacheConfig.TriesInMemory) - 1 + lastPrunedIndex := len(blocks) - int(chain.cacheConfig.TriesInMemory) - 1 lastPrunedBlock := blocks[lastPrunedIndex] - firstNonPrunedBlock := blocks[len(blocks)-int(DefaultCacheConfig.TriesInMemory)] + firstNonPrunedBlock := blocks[len(blocks)-int(chain.cacheConfig.TriesInMemory)] // Verify pruning of lastPrunedBlock if chain.HasBlockAndState(lastPrunedBlock.Hash(), lastPrunedBlock.NumberU64()) { @@ -2866,7 +2866,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - lastPrunedIndex := len(blocks) - int(DefaultCacheConfig.TriesInMemory) - 1 + lastPrunedIndex := len(blocks) - int(chain.cacheConfig.TriesInMemory) - 1 lastPrunedBlock := blocks[lastPrunedIndex] // Verify pruning of lastPrunedBlock @@ -2874,7 +2874,7 @@ func TestSideImportPrunedBlocks(t *testing.T) { t.Errorf("Block %d not pruned", lastPrunedBlock.NumberU64()) } - firstNonPrunedBlock := blocks[len(blocks)-int(DefaultCacheConfig.TriesInMemory)] + firstNonPrunedBlock := blocks[len(blocks)-int(chain.cacheConfig.TriesInMemory)] // Verify firstNonPrunedBlock is not pruned if !chain.HasBlockAndState(firstNonPrunedBlock.Hash(), firstNonPrunedBlock.NumberU64()) { t.Errorf("Block %d pruned", firstNonPrunedBlock.NumberU64()) diff --git a/core/tests/blockchain_sethead_test.go b/core/tests/blockchain_sethead_test.go index ad6e78697d..6160dfb48f 100644 --- a/core/tests/blockchain_sethead_test.go +++ b/core/tests/blockchain_sethead_test.go @@ -2082,6 +2082,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) { // verifyNoGaps checks that there are no gaps after the initial set of blocks in // the database and errors if found. +// //nolint:gocognit func verifyNoGaps(t *testing.T, chain *core.BlockChain, canonical bool, inserted types.Blocks) { t.Helper() @@ -2135,6 +2136,7 @@ func verifyNoGaps(t *testing.T, chain *core.BlockChain, canonical bool, inserted // verifyCutoff checks that there are no chain data available in the chain after // the specified limit, but that it is available before. +// //nolint:gocognit func verifyCutoff(t *testing.T, chain *core.BlockChain, canonical bool, inserted types.Blocks, head int) { t.Helper() diff --git a/eth/backend.go b/eth/backend.go index cc65d9837f..824fec8914 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -218,6 +218,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieTimeLimit: config.TrieTimeout, SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, + TriesInMemory: config.TriesInMemory, } ) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index adb36ffadd..c9272758ab 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -176,6 +176,7 @@ type Config struct { TrieTimeout time.Duration SnapshotCache int Preimages bool + TriesInMemory uint64 // Mining options Miner miner.Config From b2c125f81e633ece84895f750b40598392852bc8 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Tue, 18 Oct 2022 03:58:31 +0530 Subject: [PATCH 4/5] add : lint --- core/blockchain.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/blockchain.go b/core/blockchain.go index 5349ea8866..e1fc269759 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -230,6 +230,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par if cacheConfig == nil { cacheConfig = DefaultCacheConfig } + if cacheConfig.TriesInMemory <= 0 { cacheConfig.TriesInMemory = 128 } From 723ff9a958180f08d7a3a44849b2e57d644a524d Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 15:11:28 +0530 Subject: [PATCH 5/5] chg : minor change --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index e1fc269759..8103e4a05e 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -232,7 +232,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par } if cacheConfig.TriesInMemory <= 0 { - cacheConfig.TriesInMemory = 128 + cacheConfig.TriesInMemory = DefaultCacheConfig.TriesInMemory } bodyCache, _ := lru.New(bodyCacheLimit) bodyRLPCache, _ := lru.New(bodyCacheLimit)