diff --git a/.gitignore b/.gitignore index 7000fedd25..269455db7a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,16 @@ profile.cov .vscode tests/spec-tests/ + +# binaries +cmd/abidump/abidump +cmd/abigen/abigen +cmd/blsync/blsync +cmd/clef/clef +cmd/devp2p/devp2p +cmd/era/era +cmd/ethkey/ethkey +cmd/evm/evm +cmd/geth/geth +cmd/rlpdump/rlpdump +cmd/workload/workload \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 5e8a493d03..43f8ced19c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,6 @@ language: go go_import_path: github.com/ethereum/go-ethereum sudo: false jobs: - allow_failures: - - stage: build - os: osx - env: - - azure-osx - include: # This builder create and push the Docker images for all architectures - stage: build @@ -62,23 +56,6 @@ jobs: - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # This builder does the OSX Azure uploads - - stage: build - if: type = push - os: osx - osx_image: xcode14.2 - go: 1.23.1 # See https://github.com/ethereum/go-ethereum/pull/30478 - env: - - azure-osx - git: - submodules: false # avoid cloning ethereum/tests - script: - - ln -sf /Users/travis/gopath/bin/go1.23.1 /usr/local/bin/go # Work around travis go-setup bug - - go run build/ci.go install -dlgo - - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - - go run build/ci.go install -dlgo -arch arm64 - - go run build/ci.go archive -arch arm64 -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # These builders run the tests - stage: build if: type = push diff --git a/accounts/abi/bind/v2/base.go b/accounts/abi/bind/v2/base.go index a9b667e697..535c0ed4fd 100644 --- a/accounts/abi/bind/v2/base.go +++ b/accounts/abi/bind/v2/base.go @@ -383,13 +383,14 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad } } msg := ethereum.CallMsg{ - From: opts.From, - To: contract, - GasPrice: gasPrice, - GasTipCap: gasTipCap, - GasFeeCap: gasFeeCap, - Value: value, - Data: input, + From: opts.From, + To: contract, + GasPrice: gasPrice, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Value: value, + Data: input, + AccessList: opts.AccessList, } return c.transactor.EstimateGas(ensureContext(opts.Context), msg) } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 05c8bc4c7c..708dd8fb7a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -37,7 +37,9 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" @@ -66,7 +68,7 @@ It expects the genesis file as argument.`, Name: "dumpgenesis", Usage: "Dumps genesis block JSON configuration to stdout", ArgsUsage: "", - Flags: append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...), + Flags: slices.Concat([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags), Description: ` The dumpgenesis command prints the genesis configuration of the network preset if one is set. Otherwise it prints the genesis from the datadir.`, @@ -78,11 +80,11 @@ if one is set. Otherwise it prints the genesis from the datadir.`, ArgsUsage: " ( ... ) ", Flags: slices.Concat([]cli.Flag{ utils.CacheFlag, - utils.SyncModeFlag, utils.GCModeFlag, utils.SnapshotFlag, utils.CacheDatabaseFlag, utils.CacheGCFlag, + utils.NoCompactionFlag, utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, utils.MetricsHTTPFlag, @@ -105,7 +107,11 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, - }, utils.DatabaseFlags), + }, utils.DatabaseFlags, debug.Flags), + Before: func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) + return debug.Setup(ctx) + }, Description: ` The import command allows the import of blocks from an RLP-encoded format. This format can be a single file containing multiple RLP-encoded blocks, or multiple files can be given. @@ -119,10 +125,7 @@ to import successfully.`, Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Flags: slices.Concat([]cli.Flag{ - utils.CacheFlag, - utils.SyncModeFlag, - }, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags), Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and @@ -135,12 +138,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.TxLookupLimitFlag, - }, - utils.DatabaseFlags, - utils.NetworkFlags, - ), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -151,7 +149,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: slices.Concat(utils.DatabaseFlags), + Flags: utils.DatabaseFlags, Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -162,10 +160,7 @@ into Era archives. Eras are typically packaged in steps of 8192 blocks. Name: "import-preimages", Usage: "Import the preimage database from an RLP stream", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.CacheFlag, - utils.SyncModeFlag, - }, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags), Description: ` The import-preimages command imports hash preimages from an RLP encoded stream. It's deprecated, please use "geth db import" instead. @@ -435,6 +430,10 @@ func importHistory(ctx *cli.Context) error { network = "mainnet" case ctx.Bool(utils.SepoliaFlag.Name): network = "sepolia" + case ctx.Bool(utils.HoleskyFlag.Name): + network = "holesky" + case ctx.Bool(utils.HoodiFlag.Name): + network = "hoodi" } } else { // No network flag set, try to determine network based on files diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index cd41c57c75..44a52521f0 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -86,12 +86,10 @@ Remove blockchain and state databases`, }, } dbInspectCmd = &cli.Command{ - Action: inspect, - Name: "inspect", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: inspect, + Name: "inspect", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } @@ -109,16 +107,13 @@ a data corruption.`, Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), } dbCompactCmd = &cli.Command{ Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, utils.NetworkFlags, utils.DatabaseFlags), @@ -127,13 +122,11 @@ WARNING: This operation may take a very long time to finish, and may cause datab corruption if it is aborted during execution'!`, } dbGetCmd = &cli.Command{ - Action: dbGet, - Name: "get", - Usage: "Show the value of a database key", - ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: dbGet, + Name: "get", + Usage: "Show the value of a database key", + ArgsUsage: "", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDeleteCmd = &cli.Command{ @@ -141,9 +134,7 @@ corruption if it is aborted during execution'!`, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } @@ -152,59 +143,47 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } dbGetSlotsCmd = &cli.Command{ - Action: dbDumpTrie, - Name: "dumptrie", - Usage: "Show the storage key/values of a given storage trie", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: dbDumpTrie, + Name: "dumptrie", + Usage: "Show the storage key/values of a given storage trie", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDumpFreezerIndex = &cli.Command{ - Action: freezerInspect, - Name: "freezer-index", - Usage: "Dump out the index of a specific freezer table", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: freezerInspect, + Name: "freezer-index", + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command displays information about the freezer index.", } dbImportCmd = &cli.Command{ - Action: importLDBdata, - Name: "import", - Usage: "Imports leveldb-data from an exported RLP dump.", - ArgsUsage: " has .gz suffix, gzip compression will be used.", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: exportChaindata, + Name: "export", + Usage: "Exports the chain data into an RLP dump. If the has .gz suffix, gzip compression will be used.", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } dbMetadataCmd = &cli.Command{ - Action: showMetaData, - Name: "metadata", - Usage: "Shows metadata about the chain status.", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: showMetaData, + Name: "metadata", + Usage: "Shows metadata about the chain status.", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "Shows metadata about the chain status.", } dbInspectHistoryCmd = &cli.Command{ @@ -213,7 +192,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, Usage: "Inspect the state history within block range", ArgsUsage: "
[OPTIONAL ]", Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, &cli.Uint64Flag{ Name: "start", Usage: "block number of the range start, zero means earliest history", diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9c0c0d9dfc..cd74fb7b6a 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -142,7 +142,6 @@ var ( utils.VMTraceJsonConfigFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, - utils.NoCompactionFlag, utils.GpoBlocksFlag, utils.GpoPercentileFlag, utils.GpoMaxGasPriceFlag, diff --git a/cmd/utils/export_test.go b/cmd/utils/export_test.go index b70d2451c6..d1a004d9b7 100644 --- a/cmd/utils/export_test.go +++ b/cmd/utils/export_test.go @@ -54,8 +54,8 @@ func (iter *testIterator) Next() (byte, []byte, []byte, bool) { if iter.index == 42 { iter.index += 1 } - return OpBatchAdd, []byte(fmt.Sprintf("key-%04d", iter.index)), - []byte(fmt.Sprintf("value %d", iter.index)), true + return OpBatchAdd, fmt.Appendf(nil, "key-%04d", iter.index), + fmt.Appendf(nil, "value %d", iter.index), true } func (iter *testIterator) Release() {} @@ -72,7 +72,7 @@ func testExport(t *testing.T, f string) { } // verify for i := 0; i < 1000; i++ { - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", i)) if (i < 5 || i == 42) && err == nil { t.Fatalf("expected no element at idx %d, got '%v'", i, string(v)) } @@ -85,7 +85,7 @@ func testExport(t *testing.T, f string) { } } } - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", 1000))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", 1000)) if err == nil { t.Fatalf("expected no element at idx %d, got '%v'", 1000, string(v)) } @@ -120,7 +120,7 @@ func (iter *deletionIterator) Next() (byte, []byte, []byte, bool) { if iter.index == 42 { iter.index += 1 } - return OpBatchDel, []byte(fmt.Sprintf("key-%04d", iter.index)), nil, true + return OpBatchDel, fmt.Appendf(nil, "key-%04d", iter.index), nil, true } func (iter *deletionIterator) Release() {} @@ -132,14 +132,14 @@ func testDeletion(t *testing.T, f string) { } db := rawdb.NewMemoryDatabase() for i := 0; i < 1000; i++ { - db.Put([]byte(fmt.Sprintf("key-%04d", i)), []byte(fmt.Sprintf("value %d", i))) + db.Put(fmt.Appendf(nil, "key-%04d", i), fmt.Appendf(nil, "value %d", i)) } err = ImportLDBData(db, f, 5, make(chan struct{})) if err != nil { t.Fatal(err) } for i := 0; i < 1000; i++ { - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", i)) if i < 5 || i == 42 { if err != nil { t.Fatalf("expected element at idx %d, got '%v'", i, err) diff --git a/common/math/big.go b/common/math/big.go index 825f4baec9..493c2b7861 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -72,7 +72,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) { if i == nil { return []byte("0x0"), nil } - return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil + return fmt.Appendf(nil, "%#x", (*big.Int)(i)), nil } // Decimal256 unmarshals big.Int as a decimal string. When unmarshalling, diff --git a/common/math/integer.go b/common/math/integer.go index 25ced87053..dfcb0aecc4 100644 --- a/common/math/integer.go +++ b/common/math/integer.go @@ -48,7 +48,7 @@ func (i *HexOrDecimal64) UnmarshalText(input []byte) error { // MarshalText implements encoding.TextMarshaler. func (i HexOrDecimal64) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("%#x", uint64(i))), nil + return fmt.Appendf(nil, "%#x", uint64(i)), nil } // ParseUint64 parses s as an integer in decimal or hexadecimal syntax. diff --git a/core/chain_makers.go b/core/chain_makers.go index 8d09390b72..7a258dc05f 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -504,6 +504,13 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine if gen != nil { gen(i, b) } + + requests := b.collectRequests(false) + if requests != nil { + reqHash := types.CalcRequestsHash(requests) + b.header.RequestsHash = &reqHash + } + body := &types.Body{ Transactions: b.txs, Uncles: b.uncles, diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go index bc0de47ded..a8cf53b1c0 100644 --- a/core/filtermaps/chain_view.go +++ b/core/filtermaps/chain_view.go @@ -83,7 +83,11 @@ func (cv *ChainView) getReceipts(number uint64) types.Receipts { if number > cv.headNumber { panic("invalid block number") } - return cv.chain.GetReceiptsByHash(cv.blockHash(number)) + blockHash := cv.blockHash(number) + if blockHash == (common.Hash{}) { + log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber) + } + return cv.chain.GetReceiptsByHash(blockHash) } // limitedView returns a new chain view that is a truncated version of the parent view. diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index db7ab0a426..5722f17daa 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/log" ) @@ -59,6 +58,7 @@ type FilterMaps struct { closeCh chan struct{} closeWg sync.WaitGroup history uint64 + hashScheme bool // use hashdb-safe delete range method exportFileName string Params @@ -67,10 +67,11 @@ type FilterMaps struct { // fields written by the indexer and read by matcher backend. Indexer can // read them without a lock and write them under indexLock write lock. // Matcher backend can read them under indexLock read lock. - indexLock sync.RWMutex - indexedRange filterMapsRange - indexedView *ChainView // always consistent with the log index - hasTempRange bool + indexLock sync.RWMutex + indexedRange filterMapsRange + cleanedEpochsBefore uint32 // all unindexed data cleaned before this point + indexedView *ChainView // always consistent with the log index + hasTempRange bool // also accessed by indexer and matcher backend but no locking needed. filterMapCache *lru.Cache[uint32, filterMap] @@ -180,6 +181,10 @@ type Config struct { // This option enables the checkpoint JSON file generator. // If set, the given file will be updated with checkpoint information. ExportFileName string + + // expect trie nodes of hash based state scheme in the filtermaps key range; + // use safe iterator based implementation of DeleteRange that skips them + HashScheme bool } // NewFilterMaps creates a new FilterMaps and starts the indexer. @@ -197,6 +202,7 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f blockProcessingCh: make(chan bool, 1), history: config.History, disabled: config.Disabled, + hashScheme: config.HashScheme, disabledCh: make(chan struct{}), exportFileName: config.ExportFileName, Params: params, @@ -208,15 +214,17 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst), tailPartialEpoch: rs.TailPartialEpoch, }, - historyCutoff: historyCutoff, - finalBlock: finalBlock, - matcherSyncCh: make(chan *FilterMapsMatcherBackend), - matchers: make(map[*FilterMapsMatcherBackend]struct{}), - filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), - lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), - lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), - baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows), - renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots), + // deleting last unindexed epoch might have been interrupted by shutdown + cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1, + historyCutoff: historyCutoff, + finalBlock: finalBlock, + matcherSyncCh: make(chan *FilterMapsMatcherBackend), + matchers: make(map[*FilterMapsMatcherBackend]struct{}), + filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), + lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), + lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), + baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows), + renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots), } // Set initial indexer target. @@ -301,14 +309,24 @@ func (f *FilterMaps) reset() { // deleting the range first ensures that resetDb will be called again at next // startup and any leftover data will be removed even if it cannot finish now. rawdb.DeleteFilterMapsRange(f.db) - f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") + f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown) +} + +// isShuttingDown returns true if FilterMaps is shutting down. +func (f *FilterMaps) isShuttingDown() bool { + select { + case <-f.closeCh: + return true + default: + return false + } } // init initializes an empty log index according to the current targetView. func (f *FilterMaps) init() error { // ensure that there is no remaining data in the filter maps key range - if !f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") { - return errors.New("could not reset log index database") + if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil { + return err } f.indexLock.Lock() @@ -358,38 +376,37 @@ func (f *FilterMaps) init() error { // removeBloomBits removes old bloom bits data from the database. func (f *FilterMaps) removeBloomBits() { - f.safeDeleteRange(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database") + f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown) f.closeWg.Done() } -// safeDeleteRange calls the specified database range deleter function -// repeatedly as long as it returns leveldb.ErrTooManyKeys. -// This wrapper is necessary because of the leveldb fallback implementation -// of DeleteRange. -func (f *FilterMaps) safeDeleteRange(removeFn func(ethdb.KeyValueRangeDeleter) error, action string) bool { - start := time.Now() - var retry bool - for { - err := removeFn(f.db) - if err == nil { - if retry { - log.Info(action+" finished", "elapsed", time.Since(start)) - } - return true +// safeDeleteWithLogs is a wrapper for a function that performs a safe range +// delete operation using rawdb.SafeDeleteRange. It emits log messages if the +// process takes long enough to call the stop callback. +func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error { + var ( + start = time.Now() + logPrinted bool + lastLogPrinted = start + ) + switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { + if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) + logPrinted, lastLogPrinted = true, time.Now() } - if err != leveldb.ErrTooManyKeys { - log.Error(action+" failed", "error", err) - return false - } - select { - case <-f.closeCh: - return false - default: - } - if !retry { - log.Info(action+" in progress...", "elapsed", time.Since(start)) - retry = true + return stopCb() + }); { + case err == nil: + if logPrinted { + log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start))) } + return nil + case errors.Is(err, rawdb.ErrDeleteRangeInterrupted): + log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start))) + return err + default: + log.Error(action+" failed", "error", err) + return err } } @@ -658,54 +675,97 @@ func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) { rawdb.DeleteFilterMapLastBlock(batch, mapIndex) } -// deleteTailEpoch deletes index data from the earliest, either fully or partially -// indexed epoch. The last block pointer for the last map of the epoch and the -// corresponding block log value pointer are retained as these are always assumed -// to be available for each epoch. -func (f *FilterMaps) deleteTailEpoch(epoch uint32) error { +// deleteTailEpoch deletes index data from the specified epoch. The last block +// pointer for the last map of the epoch and the corresponding block log value +// pointer are retained as these are always assumed to be available for each +// epoch as boundary markers. +// The function returns true if all index data related to the epoch (except for +// the boundary markers) has been fully removed. +func (f *FilterMaps) deleteTailEpoch(epoch uint32) (bool, error) { f.indexLock.Lock() defer f.indexLock.Unlock() + // determine epoch boundaries firstMap := epoch << f.logMapsPerEpoch lastBlock, _, err := f.getLastBlockOfMap(firstMap + f.mapsPerEpoch - 1) if err != nil { - return fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err) + return false, fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err) } var firstBlock uint64 if epoch > 0 { firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1) if err != nil { - return fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err) + return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err) } firstBlock++ } - fmr := f.indexedRange - if f.indexedRange.maps.First() == firstMap && - f.indexedRange.maps.AfterLast() > firstMap+f.mapsPerEpoch && - f.indexedRange.tailPartialEpoch == 0 { - fmr.maps.SetFirst(firstMap + f.mapsPerEpoch) - fmr.blocks.SetFirst(lastBlock + 1) - } else if f.indexedRange.maps.First() == firstMap+f.mapsPerEpoch { + // update rendered range if necessary + var ( + fmr = f.indexedRange + firstEpoch = f.indexedRange.maps.First() >> f.logMapsPerEpoch + afterLastEpoch = (f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1) >> f.logMapsPerEpoch + ) + if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 { + firstEpoch-- + } + switch { + case epoch < firstEpoch: + // cleanup of already unindexed epoch; range not affected + case epoch == firstEpoch && epoch+1 < afterLastEpoch: + // first fully or partially rendered epoch and there is at least one + // rendered map in the next epoch; remove from indexed range fmr.tailPartialEpoch = 0 + fmr.maps.SetFirst((epoch + 1) << f.logMapsPerEpoch) + fmr.blocks.SetFirst(lastBlock + 1) + f.setRange(f.db, f.indexedView, fmr, false) + default: + // cannot be cleaned or unindexed; return with error + return false, errors.New("invalid tail epoch number") + } + // remove index data + if err := f.safeDeleteWithLogs(func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error { + first := f.mapRowIndex(firstMap, 0) + count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first + if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil { + return err + } + for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ { + f.filterMapCache.Remove(mapIndex) + } + delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry + if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil { + return err + } + for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ { + f.lastBlockCache.Remove(mapIndex) + } + delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry + if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil { + return err + } + for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ { + f.lvPointerCache.Remove(blockNumber) + } + return nil + }, fmt.Sprintf("Deleting tail epoch #%d", epoch), func() bool { + f.processEvents() + return f.stop || !f.targetHeadIndexed() + }); err == nil { + // everything removed; mark as cleaned and report success + if f.cleanedEpochsBefore == epoch { + f.cleanedEpochsBefore = epoch + 1 + } + return true, nil } else { - return errors.New("invalid tail epoch number") + // more data left in epoch range; mark as dirty and report unfinished + if f.cleanedEpochsBefore > epoch { + f.cleanedEpochsBefore = epoch + } + if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) { + return false, nil + } + return false, err } - f.setRange(f.db, f.indexedView, fmr, false) - first := f.mapRowIndex(firstMap, 0) - count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first - rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count)) - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ { - f.filterMapCache.Remove(mapIndex) - } - rawdb.DeleteFilterMapLastBlocks(f.db, common.NewRange(firstMap, f.mapsPerEpoch-1)) // keep last enrty - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ { - f.lastBlockCache.Remove(mapIndex) - } - rawdb.DeleteBlockLvPointers(f.db, common.NewRange(firstBlock, lastBlock-firstBlock)) // keep last enrty - for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ { - f.lvPointerCache.Remove(blockNumber) - } - return nil } // exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go. diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 026b3b4f38..9a5424da4a 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -17,6 +17,7 @@ package filtermaps import ( + "errors" "math" "time" @@ -49,18 +50,15 @@ func (f *FilterMaps) indexerLoop() { continue } if err := f.init(); err != nil { - log.Error("Error initializing log index; reverting to unindexed mode", "error", err) - f.reset() - f.disabled = true - close(f.disabledCh) + f.disableForError("initialization", err) + f.reset() // remove broken index from DB return } } if !f.targetHeadIndexed() { - if !f.tryIndexHead() { - // either shutdown or unexpected error; in the latter case ensure - // that proper shutdown is still possible. - f.processSingleEvent(true) + if err := f.tryIndexHead(); err != nil && err != errChainUpdate { + f.disableForError("head rendering", err) + return } } else { if f.finalBlock != f.lastFinal { @@ -69,13 +67,39 @@ func (f *FilterMaps) indexerLoop() { } f.lastFinal = f.finalBlock } - if f.tryIndexTail() && f.tryUnindexTail() { - f.waitForNewHead() + // always attempt unindexing before indexing the tail in order to + // ensure that a potentially dirty previously unindexed epoch is + // always cleaned up before any new maps are rendered. + if done, err := f.tryUnindexTail(); err != nil { + f.disableForError("tail unindexing", err) + return + } else if !done { + continue } + if done, err := f.tryIndexTail(); err != nil { + f.disableForError("tail rendering", err) + return + } else if !done { + continue + } + // tail indexing/unindexing is done; if head is also indexed then + // wait here until there is a new head + f.waitForNewHead() } } } +// disableForError is called when the indexer encounters a database error, for example a +// missing receipt. We can't continue operating when the database is broken, so the +// indexer goes into disabled state. +// Note that the partial index is left in disk; maybe a client update can fix the +// issue without reindexing. +func (f *FilterMaps) disableForError(op string, err error) { + log.Error("Log index "+op+" failed, reverting to unindexed mode", "error", err) + f.disabled = true + close(f.disabledCh) +} + type targetUpdate struct { targetView *ChainView historyCutoff, finalBlock uint64 @@ -116,14 +140,15 @@ func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) { // WaitIdle blocks until the indexer is in an idle state while synced up to the // latest targetView. func (f *FilterMaps) WaitIdle() { - if f.disabled { - f.closeWg.Wait() - return - } for { ch := make(chan bool) - f.waitIdleCh <- ch - if <-ch { + select { + case f.waitIdleCh <- ch: + if <-ch { + return + } + case <-f.disabledCh: + f.closeWg.Wait() return } } @@ -196,16 +221,16 @@ func (f *FilterMaps) setTarget(target targetUpdate) { f.finalBlock = target.finalBlock } -// tryIndexHead tries to render head maps according to the current targetView -// and returns true if successful. -func (f *FilterMaps) tryIndexHead() bool { +// tryIndexHead tries to render head maps according to the current targetView. +// Should be called when targetHeadIndexed returns false. If this function +// returns no error then either stop is true or head indexing is finished. +func (f *FilterMaps) tryIndexHead() error { headRenderer, err := f.renderMapsBefore(math.MaxUint32) if err != nil { - log.Error("Error creating log index head renderer", "error", err) - return false + return err } if headRenderer == nil { - return true + return errors.New("head indexer has nothing to do") // tryIndexHead should be called when head is not indexed } if !f.startedHeadIndex { f.lastLogHeadIndex = time.Now() @@ -230,8 +255,7 @@ func (f *FilterMaps) tryIndexHead() bool { f.lastLogHeadIndex = time.Now() } }); err != nil { - log.Error("Log index head rendering failed", "error", err) - return false + return err } if f.loggedHeadIndex && f.indexedRange.hasIndexedBlocks() { log.Info("Log index head rendering finished", @@ -240,7 +264,7 @@ func (f *FilterMaps) tryIndexHead() bool { "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) } f.loggedHeadIndex, f.startedHeadIndex = false, false - return true + return nil } // tryIndexTail tries to render tail epochs until the tail target block is @@ -248,7 +272,7 @@ func (f *FilterMaps) tryIndexHead() bool { // Note that tail indexing is only started if the log index head is fully // rendered according to targetView and is suspended as soon as the targetView // is changed. -func (f *FilterMaps) tryIndexTail() bool { +func (f *FilterMaps) tryIndexTail() (bool, error) { for { firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch if firstEpoch == 0 || !f.needTailEpoch(firstEpoch-1) { @@ -256,7 +280,7 @@ func (f *FilterMaps) tryIndexTail() bool { } f.processEvents() if f.stop || !f.targetHeadIndexed() { - return false + return false, nil } // resume process if tail rendering was interrupted because of head rendering tailRenderer := f.tailRenderer @@ -268,8 +292,7 @@ func (f *FilterMaps) tryIndexTail() bool { var err error tailRenderer, err = f.renderMapsBefore(f.indexedRange.maps.First()) if err != nil { - log.Error("Error creating log index tail renderer", "error", err) - return false + return false, err } } if tailRenderer == nil { @@ -302,13 +325,16 @@ func (f *FilterMaps) tryIndexTail() bool { f.lastLogTailIndex = time.Now() } }) - if err != nil && f.needTailEpoch(firstEpoch-1) { + if err != nil && !f.needTailEpoch(firstEpoch-1) { // stop silently if cutoff point has move beyond epoch boundary while rendering - log.Error("Log index tail rendering failed", "error", err) + return true, nil + } + if err != nil { + return false, err } if !done { f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb - return false + return false, nil } } if f.loggedTailIndex && f.indexedRange.hasIndexedBlocks() { @@ -318,32 +344,31 @@ func (f *FilterMaps) tryIndexTail() bool { "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt))) f.loggedTailIndex = false } - return true + return true, nil } // tryUnindexTail removes entire epochs of log index data as long as the first // fully indexed block is at least as old as the tail target. // Note that unindexing is very quick as it only removes continuous ranges of // data from the database and is also called while running head indexing. -func (f *FilterMaps) tryUnindexTail() bool { - for { - firstEpoch := (f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch) >> f.logMapsPerEpoch - if f.needTailEpoch(firstEpoch) { - break - } - f.processEvents() - if f.stop { - return false - } +func (f *FilterMaps) tryUnindexTail() (bool, error) { + firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch + if f.indexedRange.tailPartialEpoch > 0 && firstEpoch > 0 { + firstEpoch-- + } + for epoch := min(firstEpoch, f.cleanedEpochsBefore); !f.needTailEpoch(epoch); epoch++ { if !f.startedTailUnindex { f.startedTailUnindexAt = time.Now() f.startedTailUnindex = true f.ptrTailUnindexMap = f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch f.ptrTailUnindexBlock = f.indexedRange.blocks.First() - f.tailPartialBlocks() } - if err := f.deleteTailEpoch(firstEpoch); err != nil { - log.Error("Log index tail epoch unindexing failed", "error", err) - return false + if done, err := f.deleteTailEpoch(epoch); !done { + return false, err + } + f.processEvents() + if f.stop || !f.targetHeadIndexed() { + return false, nil } } if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() { @@ -354,7 +379,7 @@ func (f *FilterMaps) tryUnindexTail() bool { "elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt))) f.startedTailUnindex = false } - return true + return true, nil } // needTailEpoch returns true if the given tail epoch needs to be kept diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 297e339c83..c413839b7b 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -354,10 +354,8 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [] } } -func DeleteFilterMapRows(db ethdb.KeyValueRangeDeleter, mapRows common.Range[uint64]) { - if err := db.DeleteRange(filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false)); err != nil { - log.Crit("Failed to delete range of filter map rows", "err", err) - } +func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback) } // ReadFilterMapLastBlock retrieves the number of the block that generated the @@ -368,7 +366,7 @@ func ReadFilterMapLastBlock(db ethdb.KeyValueReader, mapIndex uint32) (uint64, c return 0, common.Hash{}, err } if len(enc) != 40 { - return 0, common.Hash{}, errors.New("Invalid block number and id encoding") + return 0, common.Hash{}, errors.New("invalid block number and id encoding") } var id common.Hash copy(id[:], enc[8:]) @@ -394,10 +392,8 @@ func DeleteFilterMapLastBlock(db ethdb.KeyValueWriter, mapIndex uint32) { } } -func DeleteFilterMapLastBlocks(db ethdb.KeyValueRangeDeleter, maps common.Range[uint32]) { - if err := db.DeleteRange(filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast())); err != nil { - log.Crit("Failed to delete range of filter map last block pointers", "err", err) - } +func DeleteFilterMapLastBlocks(db ethdb.KeyValueStore, maps common.Range[uint32], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast()), hashScheme, stopCallback) } // ReadBlockLvPointer retrieves the starting log value index where the log values @@ -408,7 +404,7 @@ func ReadBlockLvPointer(db ethdb.KeyValueReader, blockNumber uint64) (uint64, er return 0, err } if len(encPtr) != 8 { - return 0, errors.New("Invalid log value pointer encoding") + return 0, errors.New("invalid log value pointer encoding") } return binary.BigEndian.Uint64(encPtr), nil } @@ -431,10 +427,8 @@ func DeleteBlockLvPointer(db ethdb.KeyValueWriter, blockNumber uint64) { } } -func DeleteBlockLvPointers(db ethdb.KeyValueRangeDeleter, blocks common.Range[uint64]) { - if err := db.DeleteRange(filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast())); err != nil { - log.Crit("Failed to delete range of block log value pointers", "err", err) - } +func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast()), hashScheme, stopCallback) } // FilterMapsRange is a storage representation of the block range covered by the @@ -485,22 +479,22 @@ func DeleteFilterMapsRange(db ethdb.KeyValueWriter) { } // deletePrefixRange deletes everything with the given prefix from the database. -func deletePrefixRange(db ethdb.KeyValueRangeDeleter, prefix []byte) error { +func deletePrefixRange(db ethdb.KeyValueStore, prefix []byte, hashScheme bool, stopCallback func(bool) bool) error { end := bytes.Clone(prefix) end[len(end)-1]++ - return db.DeleteRange(prefix, end) + return SafeDeleteRange(db, prefix, end, hashScheme, stopCallback) } // DeleteFilterMapsDb removes the entire filter maps database -func DeleteFilterMapsDb(db ethdb.KeyValueRangeDeleter) error { - return deletePrefixRange(db, []byte(filterMapsPrefix)) +func DeleteFilterMapsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error { + return deletePrefixRange(db, []byte(filterMapsPrefix), hashScheme, stopCallback) } -// DeleteFilterMapsDb removes the old bloombits database and the associated +// DeleteBloomBitsDb removes the old bloombits database and the associated // chain indexer database. -func DeleteBloomBitsDb(db ethdb.KeyValueRangeDeleter) error { - if err := deletePrefixRange(db, bloomBitsPrefix); err != nil { +func DeleteBloomBitsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error { + if err := deletePrefixRange(db, bloomBitsPrefix, hashScheme, stopCallback); err != nil { return err } - return deletePrefixRange(db, bloomBitsMetaPrefix) + return deletePrefixRange(db, bloomBitsMetaPrefix, hashScheme, stopCallback) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a394f46e0a..2a50e3f9ee 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -28,12 +28,15 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) +var ErrDeleteRangeInterrupted = errors.New("safe delete range operation interrupted") + // freezerdb is a database wrapper that enables ancient chain segment freezing. type freezerdb struct { ethdb.KeyValueStore @@ -388,10 +391,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { verkleTries stat verkleStateLookups stat - // Les statistic - chtTrieNodes stat - bloomTrieNodes stat - // Meta- and unaccounted data metadata stat unaccounted stat @@ -465,16 +464,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: bloomBits.Add(size) - // LES indexes (deprecated) - case bytes.HasPrefix(key, chtTablePrefix) || - bytes.HasPrefix(key, chtIndexTablePrefix) || - bytes.HasPrefix(key, chtPrefix): // Canonical hash trie - chtTrieNodes.Add(size) - case bytes.HasPrefix(key, bloomTrieTablePrefix) || - bytes.HasPrefix(key, bloomTrieIndexPrefix) || - bytes.HasPrefix(key, bloomTriePrefix): // Bloomtrie sub - bloomTrieNodes.Add(size) - // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): remain := key[len(VerklePrefix):] @@ -538,8 +527,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, - {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, - {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, } // Inspect all registered append-only file store then. ancients, err := inspectFreezers(db) @@ -623,3 +610,73 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { } return data } + +// SafeDeleteRange deletes all of the keys (and values) in the range +// [start,end) (inclusive on start, exclusive on end). +// If hashScheme is true then it always uses an iterator and skips hashdb trie +// node entries. If it is false and the backing db is pebble db then it uses +// the fast native range delete. +// In case of fallback mode (hashdb or leveldb) the range deletion might be +// very slow depending on the number of entries. In this case stopCallback +// is periodically called and if it returns an error then SafeDeleteRange +// stops and also returns that error. The callback is not called if native +// range delete is used or there are a small number of keys only. The bool +// argument passed to the callback is true if enrties have actually been +// deleted already. +func SafeDeleteRange(db ethdb.KeyValueStore, start, end []byte, hashScheme bool, stopCallback func(bool) bool) error { + if !hashScheme { + // delete entire range; use fast native range delete on pebble db + for { + switch err := db.DeleteRange(start, end); { + case err == nil: + return nil + case errors.Is(err, ethdb.ErrTooManyKeys): + if stopCallback(true) { + return ErrDeleteRangeInterrupted + } + default: + return err + } + } + } + + var ( + count, deleted, skipped int + buff = crypto.NewKeccakState() + startTime = time.Now() + ) + + batch := db.NewBatch() + it := db.NewIterator(nil, start) + defer func() { + it.Release() // it might be replaced during the process + log.Debug("SafeDeleteRange finished", "deleted", deleted, "skipped", skipped, "elapsed", common.PrettyDuration(time.Since(startTime))) + }() + + for it.Next() && bytes.Compare(end, it.Key()) > 0 { + // Prevent deletion for trie nodes in hash mode + if len(it.Key()) != 32 || crypto.HashData(buff, it.Value()) != common.BytesToHash(it.Key()) { + if err := batch.Delete(it.Key()); err != nil { + return err + } + deleted++ + } else { + skipped++ + } + count++ + if count > 10000 { // should not block for more than a second + if err := batch.Write(); err != nil { + return err + } + if stopCallback(deleted != 0) { + return ErrDeleteRangeInterrupted + } + start = append(bytes.Clone(it.Key()), 0) // appending a zero gives us the next possible key + it.Release() + batch = db.NewBatch() + it = db.NewIterator(nil, start) + count = 0 + } + } + return batch.Write() +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 31e80da079..fa125cecc0 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -144,14 +144,6 @@ var ( // old log index bloomBitsMetaPrefix = []byte("iB") - // LES indexes - chtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash - chtTablePrefix = []byte("cht-") - chtIndexTablePrefix = []byte("chtIndexV2-") - bloomTriePrefix = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash - bloomTrieTablePrefix = []byte("blt-") - bloomTrieIndexPrefix = []byte("bltIndex-") - preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 661610840a..3de7735ef8 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -655,7 +655,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { for i := 0; i < 1000; i++ { acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) - key := hashData([]byte(fmt.Sprintf("acc-%d", i))) + key := hashData(fmt.Appendf(nil, "acc-%d", i)) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) } } diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index b9fe370b86..2e882f484e 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -329,27 +329,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers @@ -428,27 +428,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers diff --git a/core/state/statedb.go b/core/state/statedb.go index 027b83dae3..34ae303dd9 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -603,7 +603,6 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject { // Insert into the live set obj := newObject(s, addr, acct) s.setStateObject(obj) - s.AccountLoaded++ return obj } diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 7a0095a5ad..dafd185836 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -618,7 +618,7 @@ func (pool *LegacyPool) checkDelegationLimit(tx *types.Transaction) error { from, _ := types.Sender(pool.signer, tx) // validated // Short circuit if the sender has neither delegation nor pending delegation. - if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && len(pool.all.auths[from]) == 0 { + if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && pool.all.delegationTxsCount(from) == 0 { return nil } pending := pool.pending[from] @@ -1849,6 +1849,13 @@ func (t *lookup) removeAuthorities(tx *types.Transaction) { } } +// delegationTxsCount returns the number of pending authorizations for the specified address. +func (t *lookup) delegationTxsCount(addr common.Address) int { + t.lock.RLock() + defer t.lock.RUnlock() + return len(t.auths[addr]) +} + // numSlots calculates the number of slots needed for a single transaction. func numSlots(tx *types.Transaction) int { return int((tx.Size() + txSlotSize - 1) / txSlotSize) diff --git a/core/txpool/locals/tx_tracker_test.go b/core/txpool/locals/tx_tracker_test.go index cb6b9b3453..5585589b6c 100644 --- a/core/txpool/locals/tx_tracker_test.go +++ b/core/txpool/locals/tx_tracker_test.go @@ -108,6 +108,19 @@ func (env *testEnv) makeTx(nonce uint64, gasPrice *big.Int) *types.Transaction { return tx } +func (env *testEnv) makeTxs(n int) []*types.Transaction { + head := env.chain.CurrentHeader() + state, _ := env.chain.StateAt(head.Root) + nonce := state.GetNonce(address) + + var txs []*types.Transaction + for i := 0; i < n; i++ { + tx, _ := types.SignTx(types.NewTransaction(nonce+uint64(i), common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), signer, key) + txs = append(txs, tx) + } + return txs +} + func (env *testEnv) commit() { head := env.chain.CurrentBlock() block := env.chain.GetBlock(head.Hash(), head.Number.Uint64()) @@ -177,3 +190,29 @@ func TestRejectInvalids(t *testing.T) { } } } + +func TestResubmit(t *testing.T) { + env := newTestEnv(t, 10, 0, "") + defer env.close() + + txs := env.makeTxs(10) + txsA := txs[:len(txs)/2] + txsB := txs[len(txs)/2:] + env.pool.Add(txsA, true) + pending, queued := env.pool.ContentFrom(address) + if len(pending) != len(txsA) || len(queued) != 0 { + t.Fatalf("Unexpected txpool content: %d, %d", len(pending), len(queued)) + } + env.tracker.TrackAll(txs) + + resubmit, all := env.tracker.recheck(true) + if len(resubmit) != len(txsB) { + t.Fatalf("Unexpected transactions to resubmit, got: %d, want: %d", len(resubmit), len(txsB)) + } + if len(all) == 0 || len(all[address]) == 0 { + t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", 0, len(txs)) + } + if len(all[address]) != len(txs) { + t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", len(all[address]), len(txs)) + } +} diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 3c00699dc7..649c5d1a78 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -466,9 +466,9 @@ func (p *TxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) return p.subs.Track(event.JoinSubscriptions(subs...)) } -// Nonce returns the next nonce of an account, with all transactions executable +// PoolNonce returns the next nonce of an account, with all transactions executable // by the pool already applied on top. -func (p *TxPool) Nonce(addr common.Address) uint64 { +func (p *TxPool) PoolNonce(addr common.Address) uint64 { // Since (for now) accounts are unique to subpools, only one pool will have // (at max) a non-state nonce. To avoid stateful lookups, just return the // highest nonce for now. @@ -481,6 +481,15 @@ func (p *TxPool) Nonce(addr common.Address) uint64 { return nonce } +// Nonce returns the next nonce of an account at the current chain head. Unlike +// PoolNonce, this function does not account for pending executable transactions. +func (p *TxPool) Nonce(addr common.Address) uint64 { + p.stateLock.RLock() + defer p.stateLock.RUnlock() + + return p.state.GetNonce(addr) +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *TxPool) Stats() (int, int) { @@ -556,6 +565,9 @@ func (p *TxPool) Sync() error { // Clear removes all tracked txs from the subpools. func (p *TxPool) Clear() { + // Invoke Sync to ensure that txs pending addition don't get added to the pool after + // the subpools are subsequently cleared + p.Sync() for _, subpool := range p.subpools { subpool.Clear() } diff --git a/eth/api_backend.go b/eth/api_backend.go index b08e38afe5..c8c5ca707f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -327,7 +327,7 @@ func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) } func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { - return b.eth.txPool.Nonce(addr), nil + return b.eth.txPool.PoolNonce(addr), nil } func (b *EthAPIBackend) Stats() (runnable int, blocked int) { diff --git a/eth/backend.go b/eth/backend.go index 909d153a2b..ab612b1de7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -239,7 +239,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - fmConfig := filtermaps.Config{History: config.LogHistory, Disabled: config.LogNoHistory, ExportFileName: config.LogExportCheckpoints} + fmConfig := filtermaps.Config{ + History: config.LogHistory, + Disabled: config.LogNoHistory, + ExportFileName: config.LogExportCheckpoints, + HashScheme: scheme == rawdb.HashScheme, + } chainView := eth.newChainView(eth.blockchain.CurrentBlock()) historyCutoff := eth.blockchain.HistoryPruningCutoff() var finalBlock uint64 diff --git a/eth/handler.go b/eth/handler.go index 4c83f5613c..7179c9980b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -403,7 +403,7 @@ func (h *handler) unregisterPeer(id string) { // Abort if the peer does not exist peer := h.peers.peer(id) if peer == nil { - logger.Error("Ethereum peer removal failed", "err", errPeerNotRegistered) + logger.Warn("Ethereum peer removal failed", "err", errPeerNotRegistered) return } // Remove the `eth` peer if it exists diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index e8231461b0..0d51f40522 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -103,16 +103,10 @@ type AccessListTracer struct { // NewAccessListTracer creates a new tracer that can generate AccessLists. // An optional AccessList can be specified to occupy slots and addresses in // the resulting accesslist. -func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompiles []common.Address) *AccessListTracer { - excl := map[common.Address]struct{}{ - from: {}, to: {}, - } - for _, addr := range precompiles { - excl[addr] = struct{}{} - } +func NewAccessListTracer(acl types.AccessList, addressesToExclude map[common.Address]struct{}) *AccessListTracer { list := newAccessList() for _, al := range acl { - if _, ok := excl[al.Address]; !ok { + if _, ok := addressesToExclude[al.Address]; !ok { list.addAddress(al.Address) } for _, slot := range al.StorageKeys { @@ -120,7 +114,7 @@ func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompi } } return &AccessListTracer{ - excl: excl, + excl: addressesToExclude, list: list, } } diff --git a/ethdb/database.go b/ethdb/database.go index b1577512f3..f2d458b85f 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,7 +17,10 @@ // Package ethdb defines the interfaces for an Ethereum data store. package ethdb -import "io" +import ( + "errors" + "io" +) // KeyValueReader wraps the Has and Get method of a backing data store. type KeyValueReader interface { @@ -37,10 +40,14 @@ type KeyValueWriter interface { Delete(key []byte) error } +var ErrTooManyKeys = errors.New("too many keys in deleted range") + // KeyValueRangeDeleter wraps the DeleteRange method of a backing data store. type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). + // Some implementations of DeleteRange may return ErrTooManyKeys after + // partially deleting entries in the given range. DeleteRange(start, end []byte) error } diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 7f47523b82..ef02e91822 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -207,8 +207,6 @@ func (db *Database) Delete(key []byte) error { return db.db.Delete(key, nil) } -var ErrTooManyKeys = errors.New("too many keys in deleted range") - // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). // Note that this is a fallback implementation as leveldb does not natively @@ -228,7 +226,7 @@ func (db *Database) DeleteRange(start, end []byte) error { if err := batch.Write(); err != nil { return err } - return ErrTooManyKeys + return ethdb.ErrTooManyKeys } if err := batch.Delete(it.Key()); err != nil { return err diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e0d5b32622..36a5df8087 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1117,12 +1117,13 @@ type accessListResult struct { // CreateAccessList creates an EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. -func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { +// StateOverrides can be used to create the accessList while taking into account state changes from previous transactions. +func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, stateOverrides *override.StateOverride) (*accessListResult, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash } - acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args) + acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args, stateOverrides) if err != nil { return nil, err } @@ -1136,13 +1137,22 @@ func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args Transaction // AccessList creates an access list for the given transaction. // If the accesslist creation fails an error is returned. // If the transaction itself fails, an vmErr is returned. -func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { +func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs, stateOverrides *override.StateOverride) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { // Retrieve the execution context db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if db == nil || err != nil { return nil, 0, nil, err } + // Apply state overrides immediately after StateAndHeaderByNumberOrHash. + // If not applied here, there could be cases where user-specified overrides (e.g., nonce) + // may conflict with default values from the database, leading to inconsistencies. + if stateOverrides != nil { + if err := stateOverrides.Apply(db, nil); err != nil { + return nil, 0, nil, err + } + } + // Ensure any missing fields are filled, extract the recipient and input data if err = args.setFeeDefaults(ctx, b, header); err != nil { return nil, 0, nil, err @@ -1166,10 +1176,33 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Retrieve the precompiles since they don't need to be added to the access list precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number, isPostMerge, header.Time)) + // addressesToExclude contains sender, receiver, precompiles and valid authorizations + addressesToExclude := map[common.Address]struct{}{args.from(): {}, to: {}} + for _, addr := range precompiles { + addressesToExclude[addr] = struct{}{} + } + + // Prevent redundant operations if args contain more authorizations than EVM may handle + maxAuthorizations := uint64(*args.Gas) / params.CallNewAccountGas + if uint64(len(args.AuthorizationList)) > maxAuthorizations { + return nil, 0, nil, errors.New("insufficient gas to process all authorizations") + } + + for _, auth := range args.AuthorizationList { + // Duplicating stateTransition.validateAuthorization() logic + if (!auth.ChainID.IsZero() && auth.ChainID.CmpBig(b.ChainConfig().ChainID) != 0) || auth.Nonce+1 < auth.Nonce { + continue + } + + if authority, err := auth.Authority(); err == nil { + addressesToExclude[authority] = struct{}{} + } + } + // Create an initial tracer - prevTracer := logger.NewAccessListTracer(nil, args.from(), to, precompiles) + prevTracer := logger.NewAccessListTracer(nil, addressesToExclude) if args.AccessList != nil { - prevTracer = logger.NewAccessListTracer(*args.AccessList, args.from(), to, precompiles) + prevTracer = logger.NewAccessListTracer(*args.AccessList, addressesToExclude) } for { if err := ctx.Err(); err != nil { @@ -1186,7 +1219,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH msg := args.ToMessage(header.BaseFee, true, true) // Apply the transaction with the access list tracer - tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) + tracer := logger.NewAccessListTracer(accessList, addressesToExclude) config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} evm := b.GetEVM(ctx, statedb, header, &config, nil) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index b086d1a6d5..1ed1a8c8d8 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -3529,3 +3529,76 @@ func testRPCResponseWithFile(t *testing.T, testid int, result interface{}, rpc s func addressToHash(a common.Address) common.Hash { return common.BytesToHash(a.Bytes()) } + +func TestCreateAccessListWithStateOverrides(t *testing.T) { + // Initialize test backend + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7"): {Balance: big.NewInt(1000000000000000000)}, + }, + } + backend := newTestBackend(t, 1, genesis, ethash.NewFaker(), nil) + + // Create a new BlockChainAPI instance + api := NewBlockChainAPI(backend) + + // Create test contract code - a simple storage contract + // + // SPDX-License-Identifier: MIT + // pragma solidity ^0.8.0; + // + // contract SimpleStorage { + // uint256 private value; + // + // function retrieve() public view returns (uint256) { + // return value; + // } + // } + var ( + contractCode = hexutil.Bytes(common.Hex2Bytes("6080604052348015600f57600080fd5b506004361060285760003560e01c80632e64cec114602d575b600080fd5b60336047565b604051603e91906067565b60405180910390f35b60008054905090565b6000819050919050565b6061816050565b82525050565b6000602082019050607a6000830184605a565b9291505056")) + // Create state overrides with more complete state + contractAddr = common.HexToAddress("0x1234567890123456789012345678901234567890") + nonce = hexutil.Uint64(1) + overrides = &override.StateOverride{ + contractAddr: override.OverrideAccount{ + Code: &contractCode, + Balance: (*hexutil.Big)(big.NewInt(1000000000000000000)), + Nonce: &nonce, + State: map[common.Hash]common.Hash{ + common.Hash{}: common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000002a"), + }, + }, + } + ) + + // Create transaction arguments with gas and value + var ( + from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") + data = hexutil.Bytes(common.Hex2Bytes("2e64cec1")) // retrieve() + gas = hexutil.Uint64(100000) + args = TransactionArgs{ + From: &from, + To: &contractAddr, + Data: &data, + Gas: &gas, + Value: new(hexutil.Big), + } + ) + // Call CreateAccessList + result, err := api.CreateAccessList(context.Background(), args, nil, overrides) + if err != nil { + t.Fatalf("Failed to create access list: %v", err) + } + if err != nil || result == nil { + t.Fatalf("Failed to create access list: %v", err) + } + require.NotNil(t, result.Accesslist) + + // Verify access list contains the contract address and storage slot + expected := &types.AccessList{{ + Address: contractAddr, + StorageKeys: []common.Hash{{}}, + }} + require.Equal(t, expected, result.Accesslist) +} diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index 5f7af7424d..a0ddb3b29b 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -140,7 +140,7 @@ type ExtIP net.IP func (n ExtIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } -func (n ExtIP) MarshalText() ([]byte, error) { return []byte(fmt.Sprintf("extip:%v", net.IP(n))), nil } +func (n ExtIP) MarshalText() ([]byte, error) { return fmt.Appendf(nil, "extip:%v", net.IP(n)), nil } // These do nothing. diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index b8f59ee890..4a9644ac1a 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -71,7 +71,7 @@ func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) { } func (n *pmp) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("natpmp:%v", n.gw)), nil + return fmt.Appendf(nil, "natpmp:%v", n.gw), nil } func discoverPMP() Interface { diff --git a/signer/core/apitypes/signed_data_internal_test.go b/signer/core/apitypes/signed_data_internal_test.go index 1a14b35ef2..bee9f1c351 100644 --- a/signer/core/apitypes/signed_data_internal_test.go +++ b/signer/core/apitypes/signed_data_internal_test.go @@ -282,7 +282,7 @@ func TestTypedDataArrayValidate(t *testing.T) { messageHash, tErr := td.HashStruct(td.PrimaryType, td.Message) assert.NoError(t, tErr, "failed to hash message: %v", tErr) - digest := crypto.Keccak256Hash([]byte(fmt.Sprintf("%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash)))) + digest := crypto.Keccak256Hash(fmt.Appendf(nil, "%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash))) assert.Equal(t, tc.Digest, digest.String(), "digest doesn't not match") assert.NoError(t, td.validate(), "validation failed", tErr) diff --git a/signer/core/signed_data_test.go b/signer/core/signed_data_test.go index b6c080736c..001f6b6838 100644 --- a/signer/core/signed_data_test.go +++ b/signer/core/signed_data_test.go @@ -369,7 +369,7 @@ func sign(typedData apitypes.TypedData) ([]byte, []byte, error) { if err != nil { return nil, nil, err } - rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash))) + rawData := fmt.Appendf(nil, "\x19\x01%s%s", string(domainSeparator), string(typedDataHash)) sighash := crypto.Keccak256(rawData) return typedDataHash, sighash, nil } diff --git a/trie/committer.go b/trie/committer.go index 6c4374ccfd..0939a07abb 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -57,32 +57,26 @@ func (c *committer) commit(path []byte, n node, parallel bool) node { // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { case *shortNode: - // Commit child - collapsed := cn.copy() - // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. if _, ok := cn.Val.(*fullNode); ok { - collapsed.Val = c.commit(append(path, cn.Key...), cn.Val, false) + cn.Val = c.commit(append(path, cn.Key...), cn.Val, false) } // The key needs to be copied, since we're adding it to the // modified nodeset. - collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(path, collapsed) + cn.Key = hexToCompact(cn.Key) + hashedNode := c.store(path, cn) if hn, ok := hashedNode.(hashNode); ok { return hn } - return collapsed + return cn case *fullNode: - hashedKids := c.commitChildren(path, cn, parallel) - collapsed := cn.copy() - collapsed.Children = hashedKids - - hashedNode := c.store(path, collapsed) + c.commitChildren(path, cn, parallel) + hashedNode := c.store(path, cn) if hn, ok := hashedNode.(hashNode); ok { return hn } - return collapsed + return cn case hashNode: return cn default: @@ -92,11 +86,10 @@ func (c *committer) commit(path []byte, n node, parallel bool) node { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17]node { +func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) { var ( - wg sync.WaitGroup - nodesMu sync.Mutex - children [17]node + wg sync.WaitGroup + nodesMu sync.Mutex ) for i := 0; i < 16; i++ { child := n.Children[i] @@ -106,22 +99,21 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17] // If it's the hashed child, save the hash value directly. // Note: it's impossible that the child in range [0, 15] // is a valueNode. - if hn, ok := child.(hashNode); ok { - children[i] = hn + if _, ok := child.(hashNode); ok { continue } // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. if !parallel { - children[i] = c.commit(append(path, byte(i)), child, false) + n.Children[i] = c.commit(append(path, byte(i)), child, false) } else { wg.Add(1) go func(index int) { p := append(path, byte(index)) childSet := trienode.NewNodeSet(c.nodes.Owner) childCommitter := newCommitter(childSet, c.tracer, c.collectLeaf) - children[index] = childCommitter.commit(p, child, false) + n.Children[index] = childCommitter.commit(p, child, false) nodesMu.Lock() c.nodes.MergeSet(childSet) nodesMu.Unlock() @@ -132,11 +124,6 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17] if parallel { wg.Wait() } - // For the 17th child, it's possible the type is valuenode. - if n.Children[16] != nil { - children[16] = n.Children[16] - } - return children } // store hashes the node n and adds it to the modified nodeset. If leaf collection diff --git a/trie/hasher.go b/trie/hasher.go index 28f7f3d0c3..614640ae3a 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -53,62 +53,56 @@ func returnHasherToPool(h *hasher) { hasherPool.Put(h) } -// hash collapses a node down into a hash node, also returning a copy of the -// original node initialized with the computed hash to replace the original one. -func (h *hasher) hash(n node, force bool) (hashed node, cached node) { +// hash collapses a node down into a hash node. +func (h *hasher) hash(n node, force bool) node { // Return the cached hash if it's available if hash, _ := n.cache(); hash != nil { - return hash, n + return hash } // Trie not processed yet, walk the children switch n := n.(type) { case *shortNode: - collapsed, cached := h.hashShortNodeChildren(n) + collapsed := h.hashShortNodeChildren(n) hashed := h.shortnodeToHash(collapsed, force) - // We need to retain the possibly _not_ hashed node, in case it was too - // small to be hashed if hn, ok := hashed.(hashNode); ok { - cached.flags.hash = hn + n.flags.hash = hn } else { - cached.flags.hash = nil + n.flags.hash = nil } - return hashed, cached + return hashed case *fullNode: - collapsed, cached := h.hashFullNodeChildren(n) - hashed = h.fullnodeToHash(collapsed, force) + collapsed := h.hashFullNodeChildren(n) + hashed := h.fullnodeToHash(collapsed, force) if hn, ok := hashed.(hashNode); ok { - cached.flags.hash = hn + n.flags.hash = hn } else { - cached.flags.hash = nil + n.flags.hash = nil } - return hashed, cached + return hashed default: // Value and hash nodes don't have children, so they're left as were - return n, n + return n } } -// hashShortNodeChildren collapses the short node. The returned collapsed node -// holds a live reference to the Key, and must not be modified. -func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) { - // Hash the short node's child, caching the newly hashed subtree - collapsed, cached = n.copy(), n.copy() - // Previously, we did copy this one. We don't seem to need to actually - // do that, since we don't overwrite/reuse keys - // cached.Key = common.CopyBytes(n.Key) +// hashShortNodeChildren returns a copy of the supplied shortNode, with its child +// being replaced by either the hash or an embedded node if the child is small. +func (h *hasher) hashShortNodeChildren(n *shortNode) *shortNode { + var collapsed shortNode collapsed.Key = hexToCompact(n.Key) - // Unless the child is a valuenode or hashnode, hash it switch n.Val.(type) { case *fullNode, *shortNode: - collapsed.Val, cached.Val = h.hash(n.Val, false) + collapsed.Val = h.hash(n.Val, false) + default: + collapsed.Val = n.Val } - return collapsed, cached + return &collapsed } -func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) { - // Hash the full node's children, caching the newly hashed subtrees - cached = n.copy() - collapsed = n.copy() +// hashFullNodeChildren returns a copy of the supplied fullNode, with its child +// being replaced by either the hash or an embedded node if the child is small. +func (h *hasher) hashFullNodeChildren(n *fullNode) *fullNode { + var children [17]node if h.parallel { var wg sync.WaitGroup wg.Add(16) @@ -116,9 +110,9 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached go func(i int) { hasher := newHasher(false) if child := n.Children[i]; child != nil { - collapsed.Children[i], cached.Children[i] = hasher.hash(child, false) + children[i] = hasher.hash(child, false) } else { - collapsed.Children[i] = nilValueNode + children[i] = nilValueNode } returnHasherToPool(hasher) wg.Done() @@ -128,19 +122,21 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached } else { for i := 0; i < 16; i++ { if child := n.Children[i]; child != nil { - collapsed.Children[i], cached.Children[i] = h.hash(child, false) + children[i] = h.hash(child, false) } else { - collapsed.Children[i] = nilValueNode + children[i] = nilValueNode } } } - return collapsed, cached + if n.Children[16] != nil { + children[16] = n.Children[16] + } + return &fullNode{flags: nodeFlag{}, Children: children} } -// shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode -// should have hex-type Key, which will be converted (without modification) -// into compact form for RLP encoding. -// If the rlp data is smaller than 32 bytes, `nil` is returned. +// shortNodeToHash computes the hash of the given shortNode. The shortNode must +// first be collapsed, with its key converted to compact form. If the RLP-encoded +// node data is smaller than 32 bytes, the node itself is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { n.encode(h.encbuf) enc := h.encodedBytes() @@ -151,8 +147,8 @@ func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { return h.hashData(enc) } -// fullnodeToHash is used to create a hashNode from a fullNode, (which -// may contain nil values) +// fullnodeToHash computes the hash of the given fullNode. If the RLP-encoded +// node data is smaller than 32 bytes, the node itself is returned. func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { n.encode(h.encbuf) enc := h.encodedBytes() @@ -203,10 +199,10 @@ func (h *hasher) hashDataTo(dst, data []byte) { func (h *hasher) proofHash(original node) (collapsed, hashed node) { switch n := original.(type) { case *shortNode: - sn, _ := h.hashShortNodeChildren(n) + sn := h.hashShortNodeChildren(n) return sn, h.shortnodeToHash(sn, false) case *fullNode: - fn, _ := h.hashFullNodeChildren(n) + fn := h.hashFullNodeChildren(n) return fn, h.fullnodeToHash(fn, false) default: // Value and hash nodes don't have children, so they're left as were diff --git a/trie/node.go b/trie/node.go index ecc2de192d..96f077ebbb 100644 --- a/trie/node.go +++ b/trie/node.go @@ -79,15 +79,19 @@ func (n *fullNode) EncodeRLP(w io.Writer) error { return eb.Flush() } -func (n *fullNode) copy() *fullNode { copy := *n; return © } -func (n *shortNode) copy() *shortNode { copy := *n; return © } - // nodeFlag contains caching-related metadata about a node. type nodeFlag struct { hash hashNode // cached hash of the node (may be nil) dirty bool // whether the node has changes that must be written to the database } +func (n nodeFlag) copy() nodeFlag { + return nodeFlag{ + hash: common.CopyBytes(n.hash), + dirty: n.dirty, + } +} + func (n *fullNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty } func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty } func (n hashNode) cache() (hashNode, bool) { return nil, true } @@ -228,7 +232,9 @@ func decodeRef(buf []byte) (node, []byte, error) { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } - n, err := decodeNode(nil, buf) + // The buffer content has already been copied or is safe to use; + // no additional copy is required. + n, err := decodeNodeUnsafe(nil, buf) return n, rest, err case kind == rlp.String && len(val) == 0: // empty node diff --git a/trie/trie.go b/trie/trie.go index ae2a7b21a2..fdb4da9be4 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -29,11 +29,11 @@ import ( "github.com/ethereum/go-ethereum/triedb/database" ) -// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on -// top of a database. Whenever trie performs a commit operation, the generated -// nodes will be gathered and returned in a set. Once the trie is committed, -// it's not usable anymore. Callers have to re-create the trie with new root -// based on the updated trie database. +// Trie represents a Merkle Patricia Trie. Use New to create a trie that operates +// on top of a node database. During a commit operation, the trie collects all +// modified nodes into a set for return. After committing, the trie becomes +// unusable, and callers must recreate it with the new root based on the updated +// trie database. // // Trie is not safe for concurrent use. type Trie struct { @@ -67,13 +67,13 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - root: t.root, + root: copyNode(t.root), owner: t.owner, committed: t.committed, + unhashed: t.unhashed, + uncommitted: t.uncommitted, reader: t.reader, tracer: t.tracer.copy(), - uncommitted: t.uncommitted, - unhashed: t.unhashed, } } @@ -169,14 +169,12 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no } value, newnode, didResolve, err = t.get(n.Val, key, pos+len(n.Key)) if err == nil && didResolve { - n = n.copy() n.Val = newnode } return value, n, didResolve, err case *fullNode: value, newnode, didResolve, err = t.get(n.Children[key[pos]], key, pos+1) if err == nil && didResolve { - n = n.copy() n.Children[key[pos]] = newnode } return value, n, didResolve, err @@ -257,7 +255,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod } item, newnode, resolved, err = t.getNode(n.Val, path, pos+len(n.Key)) if err == nil && resolved > 0 { - n = n.copy() n.Val = newnode } return item, n, resolved, err @@ -265,7 +262,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod case *fullNode: item, newnode, resolved, err = t.getNode(n.Children[path[pos]], path, pos+1) if err == nil && resolved > 0 { - n = n.copy() n.Children[path[pos]] = newnode } return item, n, resolved, err @@ -375,7 +371,6 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error if !dirty || err != nil { return false, n, err } - n = n.copy() n.flags = t.newFlag() n.Children[key[0]] = nn return true, n, nil @@ -483,7 +478,6 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { if !dirty || err != nil { return false, n, err } - n = n.copy() n.flags = t.newFlag() n.Children[key[0]] = nn @@ -576,6 +570,36 @@ func concat(s1 []byte, s2 ...byte) []byte { return r } +// copyNode deep-copies the supplied node along with its children recursively. +func copyNode(n node) node { + switch n := (n).(type) { + case nil: + return nil + case valueNode: + return valueNode(common.CopyBytes(n)) + + case *shortNode: + return &shortNode{ + flags: n.flags.copy(), + Key: common.CopyBytes(n.Key), + Val: copyNode(n.Val), + } + case *fullNode: + var children [17]node + for i, cn := range n.Children { + children[i] = copyNode(cn) + } + return &fullNode{ + flags: n.flags.copy(), + Children: children, + } + case hashNode: + return n + default: + panic(fmt.Sprintf("%T: unknown node type", n)) + } +} + func (t *Trie) resolve(n node, prefix []byte) (node, error) { if n, ok := n.(hashNode); ok { return t.resolveAndTrack(n, prefix) @@ -593,15 +617,16 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { return nil, err } t.tracer.onRead(prefix, blob) - return mustDecodeNode(n, blob), nil + + // The returned node blob won't be changed afterward. No need to + // deep-copy the slice. + return decodeNodeUnsafe(n, blob) } // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - hash, cached := t.hashRoot() - t.root = cached - return common.BytesToHash(hash.(hashNode)) + return common.BytesToHash(t.hashRoot().(hashNode)) } // Commit collects all dirty nodes in the trie and replaces them with the @@ -652,9 +677,9 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { } // hashRoot calculates the root hash of the given trie -func (t *Trie) hashRoot() (node, node) { +func (t *Trie) hashRoot() node { if t.root == nil { - return hashNode(types.EmptyRootHash.Bytes()), nil + return hashNode(types.EmptyRootHash.Bytes()) } // If the number of changes is below 100, we let one thread handle it h := newHasher(t.unhashed >= 100) @@ -662,8 +687,7 @@ func (t *Trie) hashRoot() (node, node) { returnHasherToPool(h) t.unhashed = 0 }() - hashed, cached := h.hash(t.root, true) - return hashed, cached + return h.hash(t.root, true) } // Witness returns a set containing all trie nodes that have been accessed. diff --git a/trie/trie_test.go b/trie/trie_test.go index 77234d9d9b..54d1b083d8 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -1330,3 +1330,171 @@ func printSet(set *trienode.NodeSet) string { } return out.String() } + +func TestTrieCopy(t *testing.T) { + testTrieCopy(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopy(t, entries) +} + +func testTrieCopy(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + trCpy := tr.Copy() + + if tr.Hash() != trCpy.Hash() { + t.Errorf("Hash mismatch: old %v, copy %v", tr.Hash(), trCpy.Hash()) + } + + // Check iterator + it, _ := tr.NodeIterator(nil) + itCpy, _ := trCpy.NodeIterator(nil) + + for it.Next(false) { + hasNext := itCpy.Next(false) + if !hasNext { + t.Fatal("Iterator is not matched") + } + if !bytes.Equal(it.Path(), itCpy.Path()) { + t.Fatal("Iterator is not matched") + } + if it.Leaf() != itCpy.Leaf() { + t.Fatal("Iterator is not matched") + } + if it.Leaf() && !bytes.Equal(it.LeafBlob(), itCpy.LeafBlob()) { + t.Fatal("Iterator is not matched") + } + } + + // Check commit + root, nodes := tr.Commit(false) + rootCpy, nodesCpy := trCpy.Commit(false) + if root != rootCpy { + t.Fatal("root mismatch") + } + if len(nodes.Nodes) != len(nodesCpy.Nodes) { + t.Fatal("commit node mismatch") + } + for p, n := range nodes.Nodes { + nn, exists := nodesCpy.Nodes[p] + if !exists { + t.Fatalf("node not exists: %v", p) + } + if !reflect.DeepEqual(n, nn) { + t.Fatalf("node mismatch: %v", p) + } + } +} + +func TestTrieCopyOldTrie(t *testing.T) { + testTrieCopyOldTrie(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopyOldTrie(t, entries) +} + +func testTrieCopyOldTrie(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + hash := tr.Hash() + + trCpy := tr.Copy() + for _, val := range entries { + if rand.Intn(2) == 0 { + trCpy.Delete(val.k) + } else { + trCpy.Update(val.k, testrand.Bytes(32)) + } + } + for i := 0; i < 10; i++ { + trCpy.Update(testrand.Bytes(32), testrand.Bytes(32)) + } + trCpy.Hash() + trCpy.Commit(false) + + // Traverse the original tree, the changes made on the copy one shouldn't + // affect the old one + for _, entry := range entries { + d, _ := tr.Get(entry.k) + if !bytes.Equal(d, entry.v) { + t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d) + } + } + if tr.Hash() != hash { + t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash()) + } +} + +func TestTrieCopyNewTrie(t *testing.T) { + testTrieCopyNewTrie(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopyNewTrie(t, entries) +} + +func testTrieCopyNewTrie(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + trCpy := tr.Copy() + hash := trCpy.Hash() + + for _, val := range entries { + if rand.Intn(2) == 0 { + tr.Delete(val.k) + } else { + tr.Update(val.k, testrand.Bytes(32)) + } + } + for i := 0; i < 10; i++ { + tr.Update(testrand.Bytes(32), testrand.Bytes(32)) + } + + // Traverse the original tree, the changes made on the copy one shouldn't + // affect the old one + for _, entry := range entries { + d, _ := trCpy.Get(entry.k) + if !bytes.Equal(d, entry.v) { + t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d) + } + } + if trCpy.Hash() != hash { + t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash()) + } +} diff --git a/triedb/database/database.go b/triedb/database/database.go index cd7ec1d931..8c61ea0293 100644 --- a/triedb/database/database.go +++ b/triedb/database/database.go @@ -27,6 +27,8 @@ type NodeReader interface { // node path and the corresponding node hash. No error will be returned // if the node is not found. // + // The returned node content won't be changed after the call. + // // Don't modify the returned byte slice since it's not deep-copied and // still be referenced by database. Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) diff --git a/triedb/pathdb/iterator_test.go b/triedb/pathdb/iterator_test.go index 05a166d1b6..3894118034 100644 --- a/triedb/pathdb/iterator_test.go +++ b/triedb/pathdb/iterator_test.go @@ -371,27 +371,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers @@ -480,27 +480,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers diff --git a/version/version.go b/version/version.go index 44740849bb..2098af17b5 100644 --- a/version/version.go +++ b/version/version.go @@ -19,6 +19,6 @@ package version const ( Major = 1 // Major version component of the current release Minor = 15 // Minor version component of the current release - Patch = 6 // Patch version component of the current release + Patch = 7 // Patch version component of the current release Meta = "stable" // Version metadata to append to the version string )