diff --git a/.gitignore b/.gitignore index e24e1d1677..3f27cdc00f 100644 --- a/.gitignore +++ b/.gitignore @@ -47,4 +47,6 @@ profile.cov /dashboard/assets/package-lock.json **/yarn-error.log -logs/ \ No newline at end of file +logs/ + +tests/spec-tests/ diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index dbdcd17823..8549976480 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -606,8 +606,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) { return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") } - head := b.blockchain.CurrentHeader() - if !b.blockchain.Config().IsLondon(head.Number) { + if !b.blockchain.Config().IsLondon(header.Number) { // If there's no basefee, then it must be a non-1559 execution if call.GasPrice == nil { call.GasPrice = new(big.Int) @@ -629,13 +628,13 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM // Backfill the legacy gasPrice for EVM execution, unless we're all zeroes call.GasPrice = new(big.Int) if call.GasFeeCap.BitLen() > 0 || call.GasTipCap.BitLen() > 0 { - call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, head.BaseFee), call.GasFeeCap) + call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, header.BaseFee), call.GasFeeCap) } } } // Ensure message is initialized properly. if call.Gas == 0 { - call.Gas = 50000000 + call.Gas = 10 * header.GasLimit } if call.Value == nil { call.Value = new(big.Int) diff --git a/accounts/scwallet/README.md b/accounts/scwallet/README.md index 4313d9c6b2..28079c4743 100644 --- a/accounts/scwallet/README.md +++ b/accounts/scwallet/README.md @@ -8,7 +8,7 @@ ## Preparing the smartcard - **WARNING: FOILLOWING THESE INSTRUCTIONS WILL DESTROY THE MASTER KEY ON YOUR CARD. ONLY PROCEED IF NO FUNDS ARE ASSOCIATED WITH THESE ACCOUNTS** + **WARNING: FOLLOWING THESE INSTRUCTIONS WILL DESTROY THE MASTER KEY ON YOUR CARD. ONLY PROCEED IF NO FUNDS ARE ASSOCIATED WITH THESE ACCOUNTS** You can use status' [keycard-cli](https://github.com/status-im/keycard-cli) and you should get _at least_ version 2.1.1 of their [smartcard application](https://github.com/status-im/status-keycard/releases/download/2.2.1/keycard_v2.2.1.cap) diff --git a/build/checksums.txt b/build/checksums.txt index 17fb6dce27..a15cf09b46 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,25 +1,30 @@ # This file contains sha256 checksums of optional build dependencies. +# version:spec-tests 1.0.2 # https://github.com/ethereum/execution-spec-tests/releases +# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.2/ 24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81 fixtures.tar.gz +# version:golang 1.21.3 # https://go.dev/dl/ -bfa36bf75e9a1e9cbbdb9abcf9d1707e479bd3a07880a8ae3564caee5711cb99 go1.21.1.src.tar.gz -809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19 go1.21.1.darwin-amd64.tar.gz -ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1 go1.21.1.darwin-arm64.tar.gz -9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01 go1.21.1.freebsd-386.tar.gz -2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1 go1.21.1.freebsd-amd64.tar.gz -b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901 go1.21.1.linux-386.tar.gz -b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae go1.21.1.linux-amd64.tar.gz -7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967 go1.21.1.linux-arm64.tar.gz -f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85 go1.21.1.linux-armv6l.tar.gz -eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7 go1.21.1.linux-ppc64le.tar.gz -a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d go1.21.1.linux-s390x.tar.gz -170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b go1.21.1.windows-386.zip -10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6 go1.21.1.windows-amd64.zip -41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef go1.21.1.windows-arm64.zip +186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488 go1.21.3.src.tar.gz +27014fc69e301d7588a169ca239b3cc609f0aa1abf38528bf0d20d3b259211eb go1.21.3.darwin-amd64.tar.gz +65302a7a9f7a4834932b3a7a14cb8be51beddda757b567a2f9e0cbd0d7b5a6ab go1.21.3.darwin-arm64.tar.gz +8e0cd2f66cf1bde9d07b4aee01e3d7c3cfdd14e20650488e1683da4b8492594a go1.21.3.freebsd-386.tar.gz +6e74f65f586e93d1f3947894766f69e9b2ebda488592a09df61f36f06bfe58a8 go1.21.3.freebsd-amd64.tar.gz +fb209fd070db500a84291c5a95251cceeb1723e8f6142de9baca5af70a927c0e go1.21.3.linux-386.tar.gz +1241381b2843fae5a9707eec1f8fb2ef94d827990582c7c7c32f5bdfbfd420c8 go1.21.3.linux-amd64.tar.gz +fc90fa48ae97ba6368eecb914343590bbb61b388089510d0c56c2dde52987ef3 go1.21.3.linux-arm64.tar.gz +a1ddcaaf0821a12a800884c14cb4268ce1c1f5a0301e9060646f1e15e611c6c7 go1.21.3.linux-armv6l.tar.gz +3b0e10a3704f164a6e85e0377728ec5fd21524fabe4c925610e34076586d5826 go1.21.3.linux-ppc64le.tar.gz +4c78e2e6f4c684a3d5a9bdc97202729053f44eb7be188206f0627ef3e18716b6 go1.21.3.linux-s390x.tar.gz +e36737f4f2fadb4d2f919ec4ce517133a56e06064cca6e82fc883bb000c4d56c go1.21.3.windows-386.zip +27c8daf157493f288d42a6f38debc6a2cb391f6543139eba9152fceca0be2a10 go1.21.3.windows-amd64.zip +bfb7a5c56f9ded07d8ae0e0b3702ac07b65e68fa8f33da24ed6df4ce01fe2c5c go1.21.3.windows-arm64.zip -# https://github.com/golangci/golangci-lint/releases +# version:golangci 1.51.1 +# https://github.com/golangci/golangci-lint/releases/ +# https://github.com/golangci/golangci-lint/releases/download/v1.51.1/ fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc golangci-lint-1.51.1-darwin-amd64.tar.gz 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2 golangci-lint-1.51.1-darwin-arm64.tar.gz e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414 golangci-lint-1.51.1-freebsd-386.tar.gz @@ -48,4 +53,12 @@ bce02f7232723cb727755ee11f168a700a00896a25d37f87c4b173bce55596b4 golangci-lint- cf6403f84707ce8c98664736772271bc8874f2e760c2fd0f00cf3e85963507e9 golangci-lint-1.51.1-windows-armv7.zip # This is the builder on PPA that will build Go itself (inception-y), don't modify! +# +# This version is fine to be old and full of security holes, we just use it +# to build the latest Go. Don't change it. If it ever becomes insufficient, +# we need to switch over to a recursive builder to jump across supported +# versions. +# +# version:ppa-builder 1.19.6 +# https://go.dev/dl/ d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767 go1.19.6.src.tar.gz diff --git a/build/ci.go b/build/ci.go index 400c8bdd68..869ca07d11 100644 --- a/build/ci.go +++ b/build/ci.go @@ -136,23 +136,6 @@ var ( "golang-go": "/usr/lib/go", } - // This is the version of Go that will be downloaded by - // - // go run ci.go install -dlgo - dlgoVersion = "1.21.1" - - // This is the version of Go that will be used to bootstrap the PPA builder. - // - // This version is fine to be old and full of security holes, we just use it - // to build the latest Go. Don't change it. If it ever becomes insufficient, - // we need to switch over to a recursive builder to jumpt across supported - // versions. - gobootVersion = "1.19.6" - - // This is the version of execution-spec-tests that we are using. - // When updating, you must also update build/checksums.txt. - executionSpecTestsVersion = "1.0.2" - // This is where the tests should be unpacked. executionSpecTestsDir = "tests/spec-tests" ) @@ -192,6 +175,8 @@ func main() { doWindowsInstaller(os.Args[2:]) case "purge": doPurge(os.Args[2:]) + case "sanitycheck": + doSanityCheck() default: log.Fatal("unknown command ", os.Args[1]) } @@ -213,9 +198,8 @@ func doInstall(cmdline []string) { tc := build.GoToolchain{GOARCH: *arch, CC: *cc} if *dlgo { csdb := build.MustLoadChecksums("build/checksums.txt") - tc.Root = build.DownloadGo(csdb, dlgoVersion) + tc.Root = build.DownloadGo(csdb) } - // Disable CLI markdown doc generation in release builds. buildTags := []string{"urfave_cli_no_docs"} @@ -312,7 +296,7 @@ func doTest(cmdline []string) { // Configure the toolchain. tc := build.GoToolchain{GOARCH: *arch, CC: *cc} if *dlgo { - tc.Root = build.DownloadGo(csdb, dlgoVersion) + tc.Root = build.DownloadGo(csdb) } gotest := tc.Go("test") @@ -345,6 +329,10 @@ func doTest(cmdline []string) { // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures. func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string { + executionSpecTestsVersion, err := build.Version(csdb, "spec-tests") + if err != nil { + log.Fatal(err) + } ext := ".tar.gz" base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext) @@ -377,9 +365,11 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.51.1" - csdb := build.MustLoadChecksums("build/checksums.txt") + version, err := build.Version(csdb, "golangci") + if err != nil { + log.Fatal(err) + } arch := runtime.GOARCH ext := ".tar.gz" @@ -761,6 +751,10 @@ func doDebianSource(cmdline []string) { // to bootstrap the builder Go. func downloadGoBootstrapSources(cachedir string) string { csdb := build.MustLoadChecksums("build/checksums.txt") + gobootVersion, err := build.Version(csdb, "ppa-builder") + if err != nil { + log.Fatal(err) + } file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion) url := "https://dl.google.com/go/" + file dst := filepath.Join(cachedir, file) @@ -773,6 +767,10 @@ func downloadGoBootstrapSources(cachedir string) string { // downloadGoSources downloads the Go source tarball. func downloadGoSources(cachedir string) string { csdb := build.MustLoadChecksums("build/checksums.txt") + dlgoVersion, err := build.Version(csdb, "golang") + if err != nil { + log.Fatal(err) + } file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion) url := "https://dl.google.com/go/" + file dst := filepath.Join(cachedir, file) @@ -1099,3 +1097,7 @@ func doPurge(cmdline []string) { log.Fatal(err) } } + +func doSanityCheck() { + build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt")) +} diff --git a/cmd/clef/main.go b/cmd/clef/main.go index a8b0bf816a..06a8cd7ab7 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -1206,7 +1206,7 @@ func GenDoc(ctx *cli.Context) error { URL: accounts.URL{Path: ".. ignored .."}, }, { - Address: common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"), + Address: common.MaxAddress, }, }}) } diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index 903d754dd7..a3cc69cf19 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -114,7 +114,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) records = lrecords log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name)) - entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"}) + entries, _, err := c.ListDNSRecords(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), cloudflare.ListDNSRecordsParams{Type: "TXT"}) if err != nil { return err } @@ -141,14 +141,25 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) if path != name { ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare } - record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl} - _, err = c.CreateDNSRecord(context.Background(), c.zoneID, record) + record := cloudflare.CreateDNSRecordParams{Type: "TXT", Name: path, Content: val, TTL: ttl} + _, err = c.CreateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record) } else if old.Content != val { // Entry already exists, only change its content. log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val)) updated++ - old.Content = val - err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old) + + record := cloudflare.UpdateDNSRecordParams{ + Type: old.Type, + Name: old.Name, + Content: val, + Data: old.Data, + ID: old.ID, + Priority: old.Priority, + TTL: old.TTL, + Proxied: old.Proxied, + Tags: old.Tags, + } + _, err = c.UpdateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record) } else { skipped++ log.Debug(fmt.Sprintf("Skipping %s = %q", path, val)) @@ -168,7 +179,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) // Stale entry, nuke it. log.Debug(fmt.Sprintf("Deleting %s = %q", path, entry.Content)) deleted++ - if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil { + if err := c.DeleteDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), entry.ID); err != nil { return fmt.Errorf("failed to delete %s: %v", path, err) } } diff --git a/cmd/devp2p/internal/ethtest/snap.go b/cmd/devp2p/internal/ethtest/snap.go index f947e4bc9b..54eb63f3de 100644 --- a/cmd/devp2p/internal/ethtest/snap.go +++ b/cmd/devp2p/internal/ethtest/snap.go @@ -27,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/internal/utesting" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" ) @@ -58,7 +58,7 @@ type accRangeTest struct { func (s *Suite) TestSnapGetAccountRange(t *utesting.T) { var ( root = s.chain.RootAt(999) - ffHash = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + ffHash = common.MaxHash zero = common.Hash{} firstKeyMinus1 = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf29") firstKey = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a") @@ -125,7 +125,7 @@ type stRangesTest struct { // TestSnapGetStorageRanges various forms of GetStorageRanges requests. func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) { var ( - ffHash = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + ffHash = common.MaxHash zero = common.Hash{} firstKey = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a") secondKey = common.HexToHash("0x09e47cd5056a689e708f22fe1f932709a320518e444f5f7d8d46a3da523d6606") @@ -530,17 +530,13 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error { for i, key := range hashes { keys[i] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, len(proof)) + nodes := make(trienode.ProofList, len(proof)) for i, node := range proof { nodes[i] = node } - proofdb := nodes.NodeSet() + proofdb := nodes.Set() - var end []byte - if len(keys) > 0 { - end = keys[len(keys)-1] - } - _, err = trie.VerifyRangeProof(tc.root, tc.origin[:], end, keys, accounts, proofdb) + _, err = trie.VerifyRangeProof(tc.root, tc.origin[:], keys, accounts, proofdb) return err } diff --git a/cmd/evm/testdata/9/readme.md b/cmd/evm/testdata/9/readme.md index 5394780288..357e200682 100644 --- a/cmd/evm/testdata/9/readme.md +++ b/cmd/evm/testdata/9/readme.md @@ -1,6 +1,6 @@ ## EIP-1559 testing -This test contains testcases for EIP-1559, which uses an new transaction type and has a new block parameter. +This test contains testcases for EIP-1559, which uses a new transaction type and has a new block parameter. ### Prestate diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index aebcc29eb8..a6bb2c2d2c 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -474,7 +474,7 @@ func dump(ctx *cli.Context) error { if err != nil { return err } - triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup + triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup defer triedb.Close() state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) diff --git a/cmd/geth/genesis_test.go b/cmd/geth/genesis_test.go index 2506b42d1e..ffe8176b01 100644 --- a/cmd/geth/genesis_test.go +++ b/cmd/geth/genesis_test.go @@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) { { // Can't start pebble on top of leveldb initArgs: []string{"--db.engine", "leveldb"}, execArgs: []string{"--db.engine", "pebble"}, - execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, + execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`, }, { // Can't start leveldb on top of pebble initArgs: []string{"--db.engine", "pebble"}, execArgs: []string{"--db.engine", "leveldb"}, - execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, + execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`, }, { // Reject invalid backend choice initArgs: []string{"--db.engine", "mssql"}, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 9743a7b9ca..db226c73d8 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -268,7 +268,6 @@ var ( StateSchemeFlag = &cli.StringFlag{ Name: "state.scheme", Usage: "Scheme to use for storing ethereum state ('hash' or 'path')", - Value: rawdb.HashScheme, Category: flags.StateCategory, } StateHistoryFlag = &cli.Uint64Flag{ @@ -962,17 +961,12 @@ var ( DataDirFlag, AncientFlag, RemoteDBFlag, + DBEngineFlag, StateSchemeFlag, HttpHeaderFlag, } ) -func init() { - if rawdb.PebbleEnabled { - DatabaseFlags = append(DatabaseFlags, DBEngineFlag) - } -} - // MakeDataDir retrieves the currently requested data directory, terminating // if none (or the empty string) is specified. If the node is starting a testnet, // then a subdirectory of the specified datadir will be used. @@ -1721,15 +1715,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateHistoryFlag.Name) { cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name) } - // Parse state scheme, abort the process if it's not compatible. - chaindb := tryMakeReadOnlyDatabase(ctx, stack) - scheme, err := ParseStateScheme(ctx, chaindb) - chaindb.Close() - if err != nil { - Fatalf("%v", err) + if ctx.IsSet(StateSchemeFlag.Name) { + cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } - cfg.StateScheme = scheme - // Parse transaction history flag, if user is still using legacy config // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { @@ -2165,7 +2153,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" { Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name) } - scheme, err := ParseStateScheme(ctx, chainDb) + scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), chainDb) if err != nil { Fatalf("%v", err) } @@ -2224,47 +2212,12 @@ func MakeConsolePreloads(ctx *cli.Context) []string { return preloads } -// ParseStateScheme resolves scheme identifier from CLI flag. If the provided -// state scheme is not compatible with the one of persistent scheme, an error -// will be returned. -// -// - none: use the scheme consistent with persistent state, or fallback -// to hash-based scheme if state is empty. -// - hash: use hash-based scheme or error out if not compatible with -// persistent state scheme. -// - path: use path-based scheme or error out if not compatible with -// persistent state scheme. -func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) { - // If state scheme is not specified, use the scheme consistent - // with persistent state, or fallback to hash mode if database - // is empty. - stored := rawdb.ReadStateScheme(disk) - if !ctx.IsSet(StateSchemeFlag.Name) { - if stored == "" { - // use default scheme for empty database, flip it when - // path mode is chosen as default - log.Info("State schema set to default", "scheme", "hash") - return rawdb.HashScheme, nil - } - log.Info("State scheme set to already existing", "scheme", stored) - return stored, nil // reuse scheme of persistent scheme - } - // If state scheme is specified, ensure it's compatible with - // persistent state. - scheme := ctx.String(StateSchemeFlag.Name) - if stored == "" || scheme == stored { - log.Info("State scheme set by user", "scheme", scheme) - return scheme, nil - } - return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme) -} - // MakeTrieDatabase constructs a trie database based on the configured scheme. func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database { config := &trie.Config{ Preimages: preimage, } - scheme, err := ParseStateScheme(ctx, disk) + scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk) if err != nil { Fatalf("%v", err) } diff --git a/common/types.go b/common/types.go index 7184b2b112..aadca87f82 100644 --- a/common/types.go +++ b/common/types.go @@ -44,6 +44,12 @@ const ( var ( hashT = reflect.TypeOf(Hash{}) addressT = reflect.TypeOf(Address{}) + + // MaxAddress represents the maximum possible address value. + MaxAddress = HexToAddress("0xffffffffffffffffffffffffffffffffffffffff") + + // MaxHash represents the maximum possible hash value. + MaxHash = HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") ) // Hash represents the 32 byte Keccak256 hash of arbitrary data. diff --git a/core/genesis.go b/core/genesis.go index baace3f991..0f1e8baf43 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -120,8 +120,8 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// deriveHash computes the state root according to the genesis specification. -func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { +// hash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) hash() (common.Hash, error) { // Create an ephemeral in-memory database for computing hash, // all the derived states will be discarded to not pollute disk. db := state.NewDatabase(rawdb.NewMemoryDatabase()) @@ -142,9 +142,9 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { return statedb.Commit(0, false) } -// flush is very similar with deriveHash, but the main difference is -// all the generated states will be persisted into the given database. -// Also, the genesis state specification will be flushed as well. +// flush is very similar with hash, but the main difference is all the generated +// states will be persisted into the given database. Also, the genesis state +// specification will be flushed as well. func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { @@ -179,39 +179,6 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas return nil } -// CommitGenesisState loads the stored genesis state with the given block -// hash and commits it into the provided trie database. -func CommitGenesisState(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { - var alloc GenesisAlloc - blob := rawdb.ReadGenesisStateSpec(db, blockhash) - if len(blob) != 0 { - if err := alloc.UnmarshalJSON(blob); err != nil { - return err - } - } else { - // Genesis allocation is missing and there are several possibilities: - // the node is legacy which doesn't persist the genesis allocation or - // the persisted allocation is just lost. - // - supported networks(mainnet, testnets), recover with defined allocations - // - private network, can't recover - var genesis *Genesis - switch blockhash { - case params.MainnetGenesisHash: - genesis = DefaultGenesisBlock() - case params.GoerliGenesisHash: - genesis = DefaultGoerliGenesisBlock() - case params.SepoliaGenesisHash: - genesis = DefaultSepoliaGenesisBlock() - } - if genesis != nil { - alloc = genesis.Alloc - } else { - return errors.New("not found") - } - } - return alloc.flush(db, triedb, blockhash) -} - // GenesisAccount is an account in the state of the genesis block. type GenesisAccount struct { Code []byte `json:"code,omitempty"` @@ -444,7 +411,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { // ToBlock returns the genesis block according to genesis specification. func (g *Genesis) ToBlock() *types.Block { - root, err := g.Alloc.deriveHash() + root, err := g.Alloc.hash() if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index 6a0f2df085..fac88ff373 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -231,7 +231,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash, _ = alloc.deriveHash() + hash, _ = alloc.hash() ) blob, _ := json.Marshal(alloc) rawdb.WriteGenesisStateSpec(db, hash, blob) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index ea437b8114..78f1a70b1c 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -305,3 +305,38 @@ func ReadStateScheme(db ethdb.Reader) string { } return HashScheme } + +// ParseStateScheme checks if the specified state scheme is compatible with +// the stored state. +// +// - If the provided scheme is none, use the scheme consistent with persistent +// state, or fallback to hash-based scheme if state is empty. +// +// - If the provided scheme is hash, use hash-based scheme or error out if not +// compatible with persistent state scheme. +// +// - If the provided scheme is path: use path-based scheme or error out if not +// compatible with persistent state scheme. +func ParseStateScheme(provided string, disk ethdb.Database) (string, error) { + // If state scheme is not specified, use the scheme consistent + // with persistent state, or fallback to hash mode if database + // is empty. + stored := ReadStateScheme(disk) + if provided == "" { + if stored == "" { + // use default scheme for empty database, flip it when + // path mode is chosen as default + log.Info("State schema set to default", "scheme", "hash") + return HashScheme, nil + } + log.Info("State scheme set to already existing", "scheme", stored) + return stored, nil // reuse scheme of persistent scheme + } + // If state scheme is specified, ensure it's compatible with + // persistent state. + if stored == "" || provided == stored { + log.Info("State scheme set by user", "scheme", provided) + return provided, nil + } + return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, provided) +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 0c7cf9f11b..1d7b7d1ca8 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) @@ -321,6 +322,16 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r return NewDatabase(db), nil } +// NewPebbleDBDatabase creates a persistent key-value database without a freezer +// moving immutable chain segments into cold storage. +func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) + if err != nil { + return nil, err + } + return NewDatabase(db), nil +} + const ( dbPebble = "pebble" dbLeveldb = "leveldb" @@ -375,26 +386,16 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb) } if o.Type == dbPebble || existingDb == dbPebble { - if PebbleEnabled { - log.Info("Using pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) - } else { - return nil, errors.New("db.engine 'pebble' not supported on this platform") - } + log.Info("Using pebble as the backing database") + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } if o.Type == dbLeveldb || existingDb == dbLeveldb { log.Info("Using leveldb as the backing database") return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) } - // No pre-existing database, no user-requested one either. Default to Pebble - // on supported platforms and LevelDB on anything else. - if PebbleEnabled { - log.Info("Defaulting to pebble as the backing database") - return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) - } else { - log.Info("Defaulting to leveldb as the backing database") - return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) - } + // No pre-existing database, no user-requested one either. Default to Pebble. + log.Info("Defaulting to pebble as the backing database") + return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } // Open opens both a disk-based key-value database such as leveldb or pebble, but also diff --git a/core/rawdb/databases_64bit.go b/core/rawdb/databases_64bit.go deleted file mode 100644 index e9f9332ad0..0000000000 --- a/core/rawdb/databases_64bit.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see - -//go:build (arm64 || amd64) && !openbsd - -package rawdb - -import ( - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/pebble" -) - -// Pebble is unsupported on 32bit architecture -const PebbleEnabled = true - -// NewPebbleDBDatabase creates a persistent key-value database without a freezer -// moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) - if err != nil { - return nil, err - } - return NewDatabase(db), nil -} diff --git a/core/rawdb/databases_non64bit.go b/core/rawdb/databases_non64bit.go deleted file mode 100644 index fb0777a5e6..0000000000 --- a/core/rawdb/databases_non64bit.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !((arm64 || amd64) && !openbsd) - -package rawdb - -import ( - "errors" - - "github.com/ethereum/go-ethereum/ethdb" -) - -// Pebble is unsuported on 32bit architecture -const PebbleEnabled = false - -// NewPebbleDBDatabase creates a persistent key-value database without a freezer -// moving immutable chain segments into cold storage. -func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) { - return nil, errors.New("pebble is not supported on this platform") -} diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 1e683f76ce..321bfbc6a2 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -364,11 +364,11 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) { var nodeWriter trie.NodeWriteFunc if db != nil { - nodeWriter = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + nodeWriter = func(path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme) } } - t := trie.NewStackTrieWithOwner(nodeWriter, owner) + t := trie.NewStackTrie(nodeWriter) for leaf := range in { t.Update(leaf.key[:], leaf.value) } diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f54debebed..204584c956 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -247,11 +247,6 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker) return nil, errMissingTrie } - // Firstly find out the key of last iterated element. - var last []byte - if len(keys) > 0 { - last = keys[len(keys)-1] - } // Generate the Merkle proofs for the first and last element if origin == nil { origin = common.Hash{}.Bytes() @@ -266,9 +261,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ tr: tr, }, nil } - if last != nil { - if err := tr.Prove(last, proof); err != nil { - log.Debug("Failed to prove range", "kind", kind, "last", last, "err", err) + if len(keys) > 0 { + if err := tr.Prove(keys[len(keys)-1], proof); err != nil { + log.Debug("Failed to prove range", "kind", kind, "last", keys[len(keys)-1], "err", err) return &proofResult{ keys: keys, vals: vals, @@ -280,7 +275,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ } // Verify the snapshot segment with range prover, ensure that all flat states // in this range correspond to merkle trie. - cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof) + cont, err := trie.VerifyRangeProof(root, origin, keys, vals, proof) return &proofResult{ keys: keys, vals: vals, diff --git a/core/state/statedb.go b/core/state/statedb.go index a59de16a70..d28cd29b30 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -964,7 +964,7 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo nodes = trienode.NewNodeSet(addrHash) slots = make(map[common.Hash][]byte) ) - stack := trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { nodes.AddNode(path, trienode.NewDeleted()) size += common.StorageSize(len(path)) }) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index aade2f6d73..6c6ef5adfd 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -132,7 +132,7 @@ func TestStateProcessorErrors(t *testing.T) { ) defer blockchain.Stop() - bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) + bigNumber := new(big.Int).SetBytes(common.MaxHash.Bytes()) tooBigNumber := new(big.Int).Set(bigNumber) tooBigNumber.Add(tooBigNumber, common.Big1) for i, tt := range []struct { diff --git a/core/vm/contract.go b/core/vm/contract.go index bb0902969e..e4b03bd74f 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -31,13 +31,13 @@ type ContractRef interface { // AccountRef implements ContractRef. // // Account references are used during EVM initialisation and -// it's primary use is to fetch addresses. Removing this object +// its primary use is to fetch addresses. Removing this object // proves difficult because of the cached jump destinations which // are fetched from the parent contract (i.e. the caller), which // is a ContractRef. type AccountRef common.Address -// Address casts AccountRef to a Address +// Address casts AccountRef to an Address func (ar AccountRef) Address() common.Address { return (common.Address)(ar) } // Contract represents an ethereum contract in the state database. It contains diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 5153c8b7a3..4b141d8f9a 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -104,7 +104,7 @@ func gasSStore(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySi // Legacy rules should be applied if we are in Petersburg (removal of EIP-1283) // OR Constantinople is not active if evm.chainRules.IsPetersburg || !evm.chainRules.IsConstantinople { - // This checks for 3 scenario's and calculates gas accordingly: + // This checks for 3 scenarios and calculates gas accordingly: // // 1. From a zero-value address to a non-zero value (NEW VALUE) // 2. From a non-zero value address to a zero-value address (DELETE) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 873337850e..28da2e80e6 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -45,7 +45,7 @@ type EVMInterpreter struct { table *JumpTable hasher crypto.KeccakState // Keccak256 hasher instance shared across opcodes - hasherBuf common.Hash // Keccak256 hasher result array shared aross opcodes + hasherBuf common.Hash // Keccak256 hasher result array shared across opcodes readOnly bool // Whether to throw on stateful modifications returnData []byte // Last CALL's return data for subsequent reuse diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 6ea47d63a2..b74109da0a 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// LookupInstructionSet returns the instructionset for the fork configured by +// LookupInstructionSet returns the instruction set for the fork configured by // the rules. func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { @@ -56,7 +56,7 @@ func LookupInstructionSet(rules params.Rules) (JumpTable, error) { return newFrontierInstructionSet(), nil } -// Stack returns the mininum and maximum stack requirements. +// Stack returns the minimum and maximum stack requirements. func (op *operation) Stack() (int, int) { return op.minStack, op.maxStack } diff --git a/eth/backend.go b/eth/backend.go index af03517792..c6787870ca 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -133,8 +133,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + scheme, err := rawdb.ParseStateScheme(config.StateScheme, chainDb) + if err != nil { + return nil, err + } // Try to recover offline state pruning only in hash-based. - if config.StateScheme == rawdb.HashScheme { + if scheme == rawdb.HashScheme { if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { log.Error("Failed to recover state", "error", err) } @@ -194,7 +198,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { SnapshotLimit: config.SnapshotCache, Preimages: config.Preimages, StateHistory: config.StateHistory, - StateScheme: config.StateScheme, + StateScheme: scheme, } ) // Override the chain config with provided settings. diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 55441a2cb9..bfb1df3fb1 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/downloader" @@ -64,7 +63,6 @@ var Defaults = Config{ TxLookupLimit: 2350000, TransactionHistory: 2350000, StateHistory: params.FullImmutabilityThreshold, - StateScheme: rawdb.HashScheme, LightPeers: 100, DatabaseCache: 512, TrieCleanCache: 154, @@ -83,7 +81,7 @@ var Defaults = Config{ //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go -// Config contains configuration options for of the ETH and LES protocols. +// Config contains configuration options for ETH and LES protocols. type Config struct { // The genesis block, which is inserted if the database is empty. // If nil, the Ethereum main net block is used. @@ -105,7 +103,11 @@ type Config struct { TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. - StateScheme string `toml:",omitempty"` // State scheme used to store ethereum state and merkle trie nodes on top + + // State scheme represents the scheme used to store ethereum states and trie + // nodes on top. It can be 'hash', 'path', or none which means use the scheme + // consistent with persistent state. + StateScheme string `toml:",omitempty"` // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the @@ -177,7 +179,7 @@ func CreateConsensusEngine(config *params.ChainConfig, db ethdb.Database) (conse return beacon.New(clique.New(config.Clique, db)), nil } // If defaulting to proof-of-work, enforce an already merged network since - // we cannot run PoW algorithms and more, so we cannot even follow a chain + // we cannot run PoW algorithms anymore, so we cannot even follow a chain // not coordinated by a beacon node. if !config.TerminalTotalDifficultyPassed { return nil, errors.New("ethash is only supported as a historical component of already merged networks") diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index a11b5e2164..5747626968 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -20,6 +20,7 @@ import ( "bytes" "errors" "fmt" + "math" mrand "math/rand" "sort" "time" @@ -38,16 +39,22 @@ const ( // can announce in a short time. maxTxAnnounces = 4096 - // maxTxRetrievals is the maximum transaction number can be fetched in one - // request. The rationale to pick 256 is: - // - In eth protocol, the softResponseLimit is 2MB. Nowadays according to - // Etherscan the average transaction size is around 200B, so in theory - // we can include lots of transaction in a single protocol packet. - // - However the maximum size of a single transaction is raised to 128KB, - // so pick a middle value here to ensure we can maximize the efficiency - // of the retrieval and response size overflow won't happen in most cases. + // maxTxRetrievals is the maximum number of transactions that can be fetched + // in one request. The rationale for picking 256 is to have a reasonabe lower + // bound for the transferred data (don't waste RTTs, transfer more meaningful + // batch sizes), but also have an upper bound on the sequentiality to allow + // using our entire peerset for deliveries. + // + // This number also acts as a failsafe against malicious announces which might + // cause us to request more data than we'd expect. maxTxRetrievals = 256 + // maxTxRetrievalSize is the max number of bytes that delivered transactions + // should weigh according to the announcements. The 128KB was chosen to limit + // retrieving a maximum of one blob transaction at a time to minimize hogging + // a connection between two peers. + maxTxRetrievalSize = 128 * 1024 + // maxTxUnderpricedSetSize is the size of the underpriced transaction set that // is used to track recent transactions that have been dropped so we don't // re-request them. @@ -105,6 +112,14 @@ var ( type txAnnounce struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes being announced + metas []*txMetadata // Batch of metadatas associated with the hashes (nil before eth/68) +} + +// txMetadata is a set of extra data transmitted along the announcement for better +// fetch scheduling. +type txMetadata struct { + kind byte // Transaction consensus type + size uint32 // Transaction size in bytes } // txRequest represents an in-flight transaction retrieval request destined to @@ -120,6 +135,7 @@ type txRequest struct { type txDelivery struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes having been delivered + metas []txMetadata // Batch of metadatas associated with the delivered hashes direct bool // Whether this is a direct reply or a broadcast } @@ -155,14 +171,14 @@ type TxFetcher struct { // Stage 1: Waiting lists for newly discovered transactions that might be // broadcast without needing explicit request/reply round trips. - waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast - waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist - waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection) + waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast + waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist + waitslots map[string]map[common.Hash]*txMetadata // Waiting announcements grouped by peer (DoS protection) // Stage 2: Queue of transactions that waiting to be allocated to some peer // to be retrieved directly. - announces map[string]map[common.Hash]struct{} // Set of announced transactions, grouped by origin peer - announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash + announces map[string]map[common.Hash]*txMetadata // Set of announced transactions, grouped by origin peer + announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash // Stage 3: Set of transactions currently being retrieved, some which may be // fulfilled and some rescheduled. Note, this step shares 'announces' from the @@ -175,6 +191,7 @@ type TxFetcher struct { hasTx func(common.Hash) bool // Retrieves a tx from the local txpool addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer + dropPeer func(string) // Drops a peer in case of announcement violation step chan struct{} // Notification channel when the fetcher loop iterates clock mclock.Clock // Time wrapper to simulate in tests @@ -183,14 +200,14 @@ type TxFetcher struct { // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher { - return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, mclock.System{}, nil) +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { + return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, dropPeer, mclock.System{}, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. func NewTxFetcherForTests( - hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, + hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), clock mclock.Clock, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ notify: make(chan *txAnnounce), @@ -199,8 +216,8 @@ func NewTxFetcherForTests( quit: make(chan struct{}), waitlist: make(map[common.Hash]map[string]struct{}), waittime: make(map[common.Hash]mclock.AbsTime), - waitslots: make(map[string]map[common.Hash]struct{}), - announces: make(map[string]map[common.Hash]struct{}), + waitslots: make(map[string]map[common.Hash]*txMetadata), + announces: make(map[string]map[common.Hash]*txMetadata), announced: make(map[common.Hash]map[string]struct{}), fetching: make(map[common.Hash]string), requests: make(map[string]*txRequest), @@ -209,6 +226,7 @@ func NewTxFetcherForTests( hasTx: hasTx, addTxs: addTxs, fetchTxs: fetchTxs, + dropPeer: dropPeer, clock: clock, rand: rand, } @@ -216,7 +234,7 @@ func NewTxFetcherForTests( // Notify announces the fetcher of the potential availability of a new batch of // transactions in the network. -func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { +func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error { // Keep track of all the announced transactions txAnnounceInMeter.Mark(int64(len(hashes))) @@ -226,28 +244,35 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // still valuable to check here because it runs concurrent to the internal // loop, so anything caught here is time saved internally. var ( - unknowns = make([]common.Hash, 0, len(hashes)) + unknownHashes = make([]common.Hash, 0, len(hashes)) + unknownMetas = make([]*txMetadata, 0, len(hashes)) + duplicate int64 underpriced int64 ) - for _, hash := range hashes { + for i, hash := range hashes { switch { case f.hasTx(hash): duplicate++ case f.isKnownUnderpriced(hash): underpriced++ default: - unknowns = append(unknowns, hash) + unknownHashes = append(unknownHashes, hash) + if types == nil { + unknownMetas = append(unknownMetas, nil) + } else { + unknownMetas = append(unknownMetas, &txMetadata{kind: types[i], size: sizes[i]}) + } } } txAnnounceKnownMeter.Mark(duplicate) txAnnounceUnderpricedMeter.Mark(underpriced) // If anything's left to announce, push it into the internal loop - if len(unknowns) == 0 { + if len(unknownHashes) == 0 { return nil } - announce := &txAnnounce{origin: peer, hashes: unknowns} + announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas} select { case f.notify <- announce: return nil @@ -290,6 +315,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) // re-requesting them and dropping the peer in case of malicious transfers. var ( added = make([]common.Hash, 0, len(txs)) + metas = make([]txMetadata, 0, len(txs)) ) // proceed in batches for i := 0; i < len(txs); i += 128 { @@ -325,6 +351,10 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) otherreject++ } added = append(added, batch[j].Hash()) + metas = append(metas, txMetadata{ + kind: batch[j].Type(), + size: uint32(batch[j].Size()), + }) } knownMeter.Mark(duplicate) underpricedMeter.Mark(underpriced) @@ -337,7 +367,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) } } select { - case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: + case f.cleanup <- &txDelivery{origin: peer, hashes: added, metas: metas, direct: direct}: return nil case <-f.quit: return errTerminated @@ -394,13 +424,15 @@ func (f *TxFetcher) loop() { want := used + len(ann.hashes) if want > maxTxAnnounces { txAnnounceDOSMeter.Mark(int64(want - maxTxAnnounces)) + ann.hashes = ann.hashes[:want-maxTxAnnounces] + ann.metas = ann.metas[:want-maxTxAnnounces] } // All is well, schedule the remainder of the transactions idleWait := len(f.waittime) == 0 _, oldPeer := f.announces[ann.origin] - for _, hash := range ann.hashes { + for i, hash := range ann.hashes { // If the transaction is already downloading, add it to the list // of possible alternates (in case the current retrieval fails) and // also account it for the peer. @@ -409,9 +441,9 @@ func (f *TxFetcher) loop() { // Stage 2 and 3 share the set of origins per tx if announces := f.announces[ann.origin]; announces != nil { - announces[hash] = struct{}{} + announces[hash] = ann.metas[i] } else { - f.announces[ann.origin] = map[common.Hash]struct{}{hash: {}} + f.announces[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} } continue } @@ -422,9 +454,9 @@ func (f *TxFetcher) loop() { // Stage 2 and 3 share the set of origins per tx if announces := f.announces[ann.origin]; announces != nil { - announces[hash] = struct{}{} + announces[hash] = ann.metas[i] } else { - f.announces[ann.origin] = map[common.Hash]struct{}{hash: {}} + f.announces[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} } continue } @@ -432,12 +464,18 @@ func (f *TxFetcher) loop() { // yet downloading, add the peer as an alternate origin in the // waiting list. if f.waitlist[hash] != nil { + // Ignore double announcements from the same peer. This is + // especially important if metadata is also passed along to + // prevent malicious peers flip-flopping good/bad values. + if _, ok := f.waitlist[hash][ann.origin]; ok { + continue + } f.waitlist[hash][ann.origin] = struct{}{} if waitslots := f.waitslots[ann.origin]; waitslots != nil { - waitslots[hash] = struct{}{} + waitslots[hash] = ann.metas[i] } else { - f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: {}} + f.waitslots[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} } continue } @@ -446,9 +484,9 @@ func (f *TxFetcher) loop() { f.waittime[hash] = f.clock.Now() if waitslots := f.waitslots[ann.origin]; waitslots != nil { - waitslots[hash] = struct{}{} + waitslots[hash] = ann.metas[i] } else { - f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: {}} + f.waitslots[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} } } // If a new item was added to the waitlist, schedule it into the fetcher @@ -474,9 +512,9 @@ func (f *TxFetcher) loop() { f.announced[hash] = f.waitlist[hash] for peer := range f.waitlist[hash] { if announces := f.announces[peer]; announces != nil { - announces[hash] = struct{}{} + announces[hash] = f.waitslots[peer][hash] } else { - f.announces[peer] = map[common.Hash]struct{}{hash: {}} + f.announces[peer] = map[common.Hash]*txMetadata{hash: f.waitslots[peer][hash]} } delete(f.waitslots[peer], hash) if len(f.waitslots[peer]) == 0 { @@ -545,10 +583,27 @@ func (f *TxFetcher) loop() { case delivery := <-f.cleanup: // Independent if the delivery was direct or broadcast, remove all - // traces of the hash from internal trackers - for _, hash := range delivery.hashes { + // traces of the hash from internal trackers. That said, compare any + // advertised metadata with the real ones and drop bad peers. + for i, hash := range delivery.hashes { if _, ok := f.waitlist[hash]; ok { for peer, txset := range f.waitslots { + if meta := txset[hash]; meta != nil { + if delivery.metas[i].kind != meta.kind { + log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind) + f.dropPeer(peer) + } else if delivery.metas[i].size != meta.size { + log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size) + if math.Abs(float64(delivery.metas[i].size)-float64(meta.size)) > 8 { + // Normally we should drop a peer considering this is a protocol violation. + // However, due to the RLP vs consensus format messyness, allow a few bytes + // wiggle-room where we only warn, but don't drop. + // + // TODO(karalabe): Get rid of this relaxation when clients are proven stable. + f.dropPeer(peer) + } + } + } delete(txset, hash) if len(txset) == 0 { delete(f.waitslots, peer) @@ -558,6 +613,22 @@ func (f *TxFetcher) loop() { delete(f.waittime, hash) } else { for peer, txset := range f.announces { + if meta := txset[hash]; meta != nil { + if delivery.metas[i].kind != meta.kind { + log.Warn("Announced transaction type mismatch", "peer", peer, "tx", hash, "type", delivery.metas[i].kind, "ann", meta.kind) + f.dropPeer(peer) + } else if delivery.metas[i].size != meta.size { + log.Warn("Announced transaction size mismatch", "peer", peer, "tx", hash, "size", delivery.metas[i].size, "ann", meta.size) + if math.Abs(float64(delivery.metas[i].size)-float64(meta.size)) > 8 { + // Normally we should drop a peer considering this is a protocol violation. + // However, due to the RLP vs consensus format messyness, allow a few bytes + // wiggle-room where we only warn, but don't drop. + // + // TODO(karalabe): Get rid of this relaxation when clients are proven stable. + f.dropPeer(peer) + } + } + } delete(txset, hash) if len(txset) == 0 { delete(f.announces, peer) @@ -794,25 +865,36 @@ func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, if len(f.announces[peer]) == 0 { return // continue in the for-each } - hashes := make([]common.Hash, 0, maxTxRetrievals) - f.forEachHash(f.announces[peer], func(hash common.Hash) bool { - if _, ok := f.fetching[hash]; !ok { - // Mark the hash as fetching and stash away possible alternates - f.fetching[hash] = peer + var ( + hashes = make([]common.Hash, 0, maxTxRetrievals) + bytes uint64 + ) + f.forEachAnnounce(f.announces[peer], func(hash common.Hash, meta *txMetadata) bool { + // If the transaction is already fetching, skip to the next one + if _, ok := f.fetching[hash]; ok { + return true + } + // Mark the hash as fetching and stash away possible alternates + f.fetching[hash] = peer - if _, ok := f.alternates[hash]; ok { - panic(fmt.Sprintf("alternate tracker already contains fetching item: %v", f.alternates[hash])) - } - f.alternates[hash] = f.announced[hash] - delete(f.announced, hash) + if _, ok := f.alternates[hash]; ok { + panic(fmt.Sprintf("alternate tracker already contains fetching item: %v", f.alternates[hash])) + } + f.alternates[hash] = f.announced[hash] + delete(f.announced, hash) - // Accumulate the hash and stop if the limit was reached - hashes = append(hashes, hash) - if len(hashes) >= maxTxRetrievals { - return false // break in the for-each + // Accumulate the hash and stop if the limit was reached + hashes = append(hashes, hash) + if len(hashes) >= maxTxRetrievals { + return false // break in the for-each + } + if meta != nil { // Only set eth/68 and upwards + bytes += uint64(meta.size) + if bytes >= maxTxRetrievalSize { + return false } } - return true // continue in the for-each + return true // scheduled, try to add more }) // If any hashes were allocated, request them from the peer if len(hashes) > 0 { @@ -857,27 +939,28 @@ func (f *TxFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) } } -// forEachHash does a range loop over a map of hashes in production, but during -// testing it does a deterministic sorted random to allow reproducing issues. -func (f *TxFetcher) forEachHash(hashes map[common.Hash]struct{}, do func(hash common.Hash) bool) { +// forEachAnnounce does a range loop over a map of announcements in production, +// but during testing it does a deterministic sorted random to allow reproducing +// issues. +func (f *TxFetcher) forEachAnnounce(announces map[common.Hash]*txMetadata, do func(hash common.Hash, meta *txMetadata) bool) { // If we're running production, use whatever Go's map gives us if f.rand == nil { - for hash := range hashes { - if !do(hash) { + for hash, meta := range announces { + if !do(hash, meta) { return } } return } // We're running the test suite, make iteration deterministic - list := make([]common.Hash, 0, len(hashes)) - for hash := range hashes { + list := make([]common.Hash, 0, len(announces)) + for hash := range announces { list = append(list, hash) } sortHashes(list) rotateHashes(list, f.rand.Intn(len(list))) for _, hash := range list { - if !do(hash) { + if !do(hash, announces[hash]) { return } } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 980c1a6c26..fbb9ff9dcc 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/params" ) var ( @@ -41,9 +42,20 @@ var ( testTxsHashes = []common.Hash{testTxs[0].Hash(), testTxs[1].Hash(), testTxs[2].Hash(), testTxs[3].Hash()} ) +type announce struct { + hash common.Hash + kind *byte + size *uint32 +} + +func typeptr(t byte) *byte { return &t } +func sizeptr(n uint32) *uint32 { return &n } + type doTxNotify struct { peer string hashes []common.Hash + types []byte + sizes []uint32 } type doTxEnqueue struct { peer string @@ -57,7 +69,14 @@ type doWait struct { type doDrop string type doFunc func() +type isWaitingWithMeta map[string][]announce type isWaiting map[string][]common.Hash + +type isScheduledWithMeta struct { + tracking map[string][]announce + fetching map[string][]common.Hash + dangling map[string][]common.Hash +} type isScheduled struct { tracking map[string][]common.Hash fetching map[string][]common.Hash @@ -81,6 +100,7 @@ func TestTransactionFetcherWaiting(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -162,6 +182,212 @@ func TestTransactionFetcherWaiting(t *testing.T) { }) } +// Tests that transaction announcements with associated metadata are added to a +// waitlist, and none of them are scheduled for retrieval until the wait expires. +// +// This test is an extended version of TestTransactionFetcherWaiting. It's mostly +// to cover the metadata checkes without bloating up the basic behavioral tests +// with all the useless extra fields. +func TestTransactionFetcherWaitingWithMeta(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + nil, + ) + }, + steps: []interface{}{ + // Initial announcement to get something into the waitlist + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + }, + }), + // Announce from a new peer to check that no overwrite happens + doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{333, 444}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + }), + // Announce clashing hashes but unique new peer + doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 444}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + }), + // Announce existing and clashing hashes from existing peer. Clashes + // should not overwrite previous announcements. + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{999, 333, 555}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + }), + // Announce clashing hashes with conflicting metadata. Somebody will + // be in the wrong, but we don't know yet who. + doTxNotify{peer: "D", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.BlobTxType}, sizes: []uint32{999, 222}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "D": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, + {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + }, + }), + isScheduled{tracking: nil, fetching: nil}, + + // Wait for the arrival timeout which should move all expired items + // from the wait list to the scheduler + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduledWithMeta{ + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "D": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, + {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + }, + }, + fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer + "A": {{0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + "D": {{0x02}}, + }, + }, + // Queue up a non-fetchable transaction and then trigger it with a new + // peer (weird case to test 1 line in the fetcher) + doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, + isWaitingWithMeta(map[string][]announce{ + "C": { + {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, + {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + }, + }), + doWait{time: txArriveTimeout, step: true}, + isScheduledWithMeta{ + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, + {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + }, + "D": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, + {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + }, + }, + fetching: map[string][]common.Hash{ + "A": {{0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + "D": {{0x02}}, + }, + }, + doTxNotify{peer: "E", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, + isScheduledWithMeta{ + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + }, + "B": { + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + }, + "C": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, + {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + }, + "D": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, + {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + }, + "E": { + {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, + {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + }, + }, + fetching: map[string][]common.Hash{ + "A": {{0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + "D": {{0x02}}, + "E": {{0x06}, {0x07}}, + }, + }, + }, + }) +} + // Tests that transaction announcements skip the waiting list if they are // already scheduled. func TestTransactionFetcherSkipWaiting(t *testing.T) { @@ -171,6 +397,7 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -234,6 +461,7 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -313,6 +541,7 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { <-proceed return errors.New("peer disconnected") }, + nil, ) }, steps: []interface{}{ @@ -382,6 +611,7 @@ func TestTransactionFetcherCleanup(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -421,6 +651,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -459,6 +690,7 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -505,6 +737,7 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -543,6 +776,7 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -591,6 +825,7 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -648,6 +883,7 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -713,6 +949,7 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -757,21 +994,21 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { }) } -// Tests that if thousands of transactions are announces, only a small +// Tests that if thousands of transactions are announced, only a small // number of them will be requested at a time. func TestTransactionFetcherRateLimiting(t *testing.T) { - // Create a slew of transactions and to announce them + // Create a slew of transactions and announce them var hashes []common.Hash for i := 0; i < maxTxAnnounces; i++ { hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)}) } - testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -792,6 +1029,68 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { }) } +// Tests that if huge transactions are announced, only a small number of them will +// be requested at a time, to keep the responses below a resonable level. +func TestTransactionFetcherBandwidthLimiting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + nil, + ) + }, + steps: []interface{}{ + // Announce mid size transactions from A to verify that multiple + // ones can be piled into a single request. + doTxNotify{peer: "A", + hashes: []common.Hash{{0x01}, {0x02}, {0x03}, {0x04}}, + types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, + sizes: []uint32{48 * 1024, 48 * 1024, 48 * 1024, 48 * 1024}, + }, + // Announce exactly on the limit transactions to see that only one + // gets requested + doTxNotify{peer: "B", + hashes: []common.Hash{{0x05}, {0x06}}, + types: []byte{types.LegacyTxType, types.LegacyTxType}, + sizes: []uint32{maxTxRetrievalSize, maxTxRetrievalSize}, + }, + // Announce oversized blob transactions to see that overflows are ok + doTxNotify{peer: "C", + hashes: []common.Hash{{0x07}, {0x08}}, + types: []byte{types.BlobTxType, types.BlobTxType}, + sizes: []uint32{params.MaxBlobGasPerBlock, params.MaxBlobGasPerBlock}, + }, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduledWithMeta{ + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + }, + "B": { + {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, + {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, + }, + "C": { + {common.Hash{0x07}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, + {common.Hash{0x08}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, + }, + }, + fetching: map[string][]common.Hash{ + "A": {{0x02}, {0x03}, {0x04}}, + "B": {{0x06}}, + "C": {{0x08}}, + }, + }, + }, + }) +} + // Tests that then number of transactions a peer is allowed to announce and/or // request at the same time is hard capped. func TestTransactionFetcherDoSProtection(t *testing.T) { @@ -810,6 +1109,7 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { func(common.Hash) bool { return false }, nil, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -877,6 +1177,7 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) { return errs }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -946,6 +1247,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { return errs }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: append(steps, []interface{}{ @@ -968,6 +1270,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1021,6 +1324,7 @@ func TestTransactionFetcherDrop(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1087,6 +1391,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1120,6 +1425,74 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { }) } +// Tests that announced transactions with the wrong transaction type or size will +// result in a dropped peer. +func TestInvalidAnnounceMetadata(t *testing.T) { + drop := make(chan string, 2) + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + func(peer string) { drop <- peer }, + ) + }, + steps: []interface{}{ + // Initial announcement to get something into the waitlist + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, types: []byte{testTxs[0].Type(), testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, + {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + }, + }), + // Announce from new peers conflicting transactions + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{1024 + uint32(testTxs[0].Size())}}, + doTxNotify{peer: "C", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{1 + testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, + isWaitingWithMeta(map[string][]announce{ + "A": { + {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, + {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + }, + "B": { + {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + }, + "C": { + {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + }, + }), + // Schedule all the transactions for retrieval + doWait{time: txArriveTimeout, step: true}, + isWaitingWithMeta(nil), + isScheduledWithMeta{ + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, + {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + }, + "B": { + {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + }, + "C": { + {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + }, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + "C": {testTxsHashes[1]}, + }, + }, + // Deliver the transactions and wait for B to be dropped + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}}, + doFunc(func() { <-drop }), + doFunc(func() { <-drop }), + }, + }) +} + // This test reproduces a crash caught by the fuzzer. The root cause was a // dangling transaction timing out and clashing on re-add with a concurrently // announced one. @@ -1132,6 +1505,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1159,6 +1533,7 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1188,6 +1563,7 @@ func TestTransactionFetcherFuzzCrash03(t *testing.T) { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, ) }, steps: []interface{}{ @@ -1224,6 +1600,7 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { <-proceed return errors.New("peer disconnected") }, + nil, ) }, steps: []interface{}{ @@ -1274,9 +1651,34 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { // Crunch through all the test steps and execute them for i, step := range tt.steps { + // Auto-expand certain steps to ones with metadata + switch old := step.(type) { + case isWaiting: + new := make(isWaitingWithMeta) + for peer, hashes := range old { + for _, hash := range hashes { + new[peer] = append(new[peer], announce{hash, nil, nil}) + } + } + step = new + + case isScheduled: + new := isScheduledWithMeta{ + tracking: make(map[string][]announce), + fetching: old.fetching, + dangling: old.dangling, + } + for peer, hashes := range old.tracking { + for _, hash := range hashes { + new.tracking[peer] = append(new.tracking[peer], announce{hash, nil, nil}) + } + } + step = new + } + // Process the original or expanded steps switch step := step.(type) { case doTxNotify: - if err := fetcher.Notify(step.peer, step.hashes); err != nil { + if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil { t.Errorf("step %d: %v", i, err) } <-wait // Fetcher needs to process this, wait until it's done @@ -1307,24 +1709,34 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { case doFunc: step() - case isWaiting: + case isWaitingWithMeta: // We need to check that the waiting list (stage 1) internals // match with the expected set. Check the peer->hash mappings // first. - for peer, hashes := range step { + for peer, announces := range step { waiting := fetcher.waitslots[peer] if waiting == nil { t.Errorf("step %d: peer %s missing from waitslots", i, peer) continue } - for _, hash := range hashes { - if _, ok := waiting[hash]; !ok { - t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, hash) + for _, ann := range announces { + if meta, ok := waiting[ann.hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, ann.hash) + } else { + if (meta == nil && (ann.kind != nil || ann.size != nil)) || + (meta != nil && (ann.kind == nil || ann.size == nil)) || + (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { + t.Errorf("step %d, peer %s, hash %x: waitslot metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + } } } - for hash := range waiting { - if !containsHash(hashes, hash) { - t.Errorf("step %d, peer %s: hash %x extra in waitslots", i, peer, hash) + for hash, meta := range waiting { + ann := announce{hash: hash} + if meta != nil { + ann.kind, ann.size = &meta.kind, &meta.size + } + if !containsAnnounce(announces, ann) { + t.Errorf("step %d, peer %s: announce %v extra in waitslots", i, peer, ann) } } } @@ -1334,13 +1746,13 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { } } // Peer->hash sets correct, check the hash->peer and timeout sets - for peer, hashes := range step { - for _, hash := range hashes { - if _, ok := fetcher.waitlist[hash][peer]; !ok { - t.Errorf("step %d, hash %x: peer %s missing from waitlist", i, hash, peer) + for peer, announces := range step { + for _, ann := range announces { + if _, ok := fetcher.waitlist[ann.hash][peer]; !ok { + t.Errorf("step %d, hash %x: peer %s missing from waitlist", i, ann.hash, peer) } - if _, ok := fetcher.waittime[hash]; !ok { - t.Errorf("step %d: hash %x missing from waittime", i, hash) + if _, ok := fetcher.waittime[ann.hash]; !ok { + t.Errorf("step %d: hash %x missing from waittime", i, ann.hash) } } } @@ -1349,15 +1761,15 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { t.Errorf("step %d, hash %x: empty peerset in waitlist", i, hash) } for peer := range peers { - if !containsHash(step[peer], hash) { + if !containsHashInAnnounces(step[peer], hash) { t.Errorf("step %d, hash %x: peer %s extra in waitlist", i, hash, peer) } } } for hash := range fetcher.waittime { var found bool - for _, hashes := range step { - if containsHash(hashes, hash) { + for _, announces := range step { + if containsHashInAnnounces(announces, hash) { found = true break } @@ -1367,23 +1779,33 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { } } - case isScheduled: + case isScheduledWithMeta: // Check that all scheduled announces are accounted for and no // extra ones are present. - for peer, hashes := range step.tracking { + for peer, announces := range step.tracking { scheduled := fetcher.announces[peer] if scheduled == nil { t.Errorf("step %d: peer %s missing from announces", i, peer) continue } - for _, hash := range hashes { - if _, ok := scheduled[hash]; !ok { - t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, hash) + for _, ann := range announces { + if meta, ok := scheduled[ann.hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash) + } else { + if (meta == nil && (ann.kind != nil || ann.size != nil)) || + (meta != nil && (ann.kind == nil || ann.size == nil)) || + (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { + t.Errorf("step %d, peer %s, hash %x: announce metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + } } } - for hash := range scheduled { - if !containsHash(hashes, hash) { - t.Errorf("step %d, peer %s: hash %x extra in announces", i, peer, hash) + for hash, meta := range scheduled { + ann := announce{hash: hash} + if meta != nil { + ann.kind, ann.size = &meta.kind, &meta.size + } + if !containsAnnounce(announces, ann) { + t.Errorf("step %d, peer %s: announce %x extra in announces", i, peer, hash) } } } @@ -1483,17 +1905,17 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { // retrieval but not actively being downloaded are tracked only // in the stage 2 `announced` map. var queued []common.Hash - for _, hashes := range step.tracking { - for _, hash := range hashes { + for _, announces := range step.tracking { + for _, ann := range announces { var found bool for _, hs := range step.fetching { - if containsHash(hs, hash) { + if containsHash(hs, ann.hash) { found = true break } } if !found { - queued = append(queued, hash) + queued = append(queued, ann.hash) } } } @@ -1526,6 +1948,42 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { } } +// containsAnnounce returns whether an announcement is contained within a slice +// of announcements. +func containsAnnounce(slice []announce, ann announce) bool { + for _, have := range slice { + if have.hash == ann.hash { + if have.kind == nil || ann.kind == nil { + if have.kind != ann.kind { + return false + } + } else if *have.kind != *ann.kind { + return false + } + if have.size == nil || ann.size == nil { + if have.size != ann.size { + return false + } + } else if *have.size != *ann.size { + return false + } + return true + } + } + return false +} + +// containsHashInAnnounces returns whether a hash is contained within a slice +// of announcements. +func containsHashInAnnounces(slice []announce, hash common.Hash) bool { + for _, have := range slice { + if have.hash == hash { + return true + } + } + return false +} + // containsHash returns whether a hash is contained within a hash slice. func containsHash(slice []common.Hash, hash common.Hash) bool { for _, have := range slice { diff --git a/eth/handler.go b/eth/handler.go index 0c0c17fee1..f0021e5644 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -277,7 +277,7 @@ func newHandler(config *handlerConfig) (*handler, error) { addTxs := func(txs []*types.Transaction) []error { return h.txpool.Add(txs, false, false) } - h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx) + h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) h.chainSync = newChainSyncer(h) return h, nil } diff --git a/eth/handler_eth.go b/eth/handler_eth.go index e844b36cca..2a839f615f 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -68,10 +68,10 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { return h.handleBlockBroadcast(peer, packet.Block, packet.TD) case *eth.NewPooledTransactionHashesPacket67: - return h.txFetcher.Notify(peer.ID(), *packet) + return h.txFetcher.Notify(peer.ID(), nil, nil, *packet) case *eth.NewPooledTransactionHashesPacket68: - return h.txFetcher.Notify(peer.ID(), packet.Hashes) + return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) case *eth.TransactionsPacket: for _, tx := range *packet { diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index b2fd03766e..bd7ce9e715 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -24,13 +24,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" ) const ( @@ -321,7 +321,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac it.Release() // Generate the Merkle proofs for the first and last account - proof := light.NewNodeSet() + proof := trienode.NewProofSet() if err := tr.Prove(req.Origin[:], proof); err != nil { log.Warn("Failed to prove account range", "origin", req.Origin, "err", err) return nil, nil @@ -333,7 +333,7 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac } } var proofs [][]byte - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } return accounts, proofs @@ -367,7 +367,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if len(req.Origin) > 0 { origin, req.Origin = common.BytesToHash(req.Origin), nil } - var limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + var limit = common.MaxHash if len(req.Limit) > 0 { limit, req.Limit = common.BytesToHash(req.Limit), nil } @@ -427,7 +427,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if err != nil { return nil, nil } - proof := light.NewNodeSet() + proof := trienode.NewProofSet() if err := stTrie.Prove(origin[:], proof); err != nil { log.Warn("Failed to prove storage range", "origin", req.Origin, "err", err) return nil, nil @@ -438,7 +438,7 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP return nil, nil } } - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } // Proof terminates the reply as proofs are only added if a node diff --git a/eth/protocols/snap/range.go b/eth/protocols/snap/range.go index 2627cb954b..8c98c71d50 100644 --- a/eth/protocols/snap/range.go +++ b/eth/protocols/snap/range.go @@ -67,7 +67,7 @@ func (r *hashRange) End() common.Hash { // If the end overflows (non divisible range), return a shorter interval next, overflow := new(uint256.Int).AddOverflow(r.current, r.step) if overflow { - return common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + return common.MaxHash } return next.SubUint64(next, 1).Bytes32() } diff --git a/eth/protocols/snap/range_test.go b/eth/protocols/snap/range_test.go index 3461439e54..ea643f1361 100644 --- a/eth/protocols/snap/range_test.go +++ b/eth/protocols/snap/range_test.go @@ -45,7 +45,7 @@ func TestHashRanges(t *testing.T) { common.HexToHash("0x3fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), common.HexToHash("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), common.HexToHash("0xbfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), - common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.MaxHash, }, }, // Split a divisible part of the hash range up into 2 chunks @@ -58,7 +58,7 @@ func TestHashRanges(t *testing.T) { }, ends: []common.Hash{ common.HexToHash("0x8fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), - common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.MaxHash, }, }, // Split the entire hash range into a non divisible 3 chunks @@ -73,7 +73,7 @@ func TestHashRanges(t *testing.T) { ends: []common.Hash{ common.HexToHash("0x5555555555555555555555555555555555555555555555555555555555555555"), common.HexToHash("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab"), - common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.MaxHash, }, }, // Split a part of hash range into a non divisible 3 chunks @@ -88,7 +88,7 @@ func TestHashRanges(t *testing.T) { ends: []common.Hash{ common.HexToHash("0x6aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), common.HexToHash("0xb555555555555555555555555555555555555555555555555555555555555555"), - common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.MaxHash, }, }, // Split a part of hash range into a non divisible 3 chunks, but with a @@ -108,7 +108,7 @@ func TestHashRanges(t *testing.T) { ends: []common.Hash{ common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff5"), common.HexToHash("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffb"), - common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + common.MaxHash, }, }, } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 0f5f2ccdfe..22638d0473 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -37,11 +37,11 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/msgrate" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" ) @@ -738,8 +738,8 @@ func (s *Syncer) loadSyncStatus() { s.accountBytes += common.StorageSize(len(key) + len(value)) }, } - task.genTrie = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - rawdb.WriteTrieNode(task.genBatch, owner, path, hash, val, s.scheme) + task.genTrie = trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { + rawdb.WriteTrieNode(task.genBatch, common.Hash{}, path, hash, val, s.scheme) }) for accountHash, subtasks := range task.SubTasks { for _, subtask := range subtasks { @@ -751,9 +751,10 @@ func (s *Syncer) loadSyncStatus() { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } - subtask.genTrie = trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { + owner := accountHash // local assignment for stacktrie writer closure + subtask.genTrie = trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { rawdb.WriteTrieNode(subtask.genBatch, owner, path, hash, val, s.scheme) - }, accountHash) + }) } } } @@ -797,7 +798,7 @@ func (s *Syncer) loadSyncStatus() { last := common.BigToHash(new(big.Int).Add(next.Big(), step)) if i == accountConcurrency-1 { // Make sure we don't overflow if the step is not a proper divisor - last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + last = common.MaxHash } batch := ethdb.HookedBatch{ Batch: s.db.NewBatch(), @@ -810,8 +811,8 @@ func (s *Syncer) loadSyncStatus() { Last: last, SubTasks: make(map[common.Hash][]*storageTask), genBatch: batch, - genTrie: trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) + genTrie: trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { + rawdb.WriteTrieNode(batch, common.Hash{}, path, hash, val, s.scheme) }), }) log.Debug("Created account sync task", "from", next, "last", last) @@ -1873,7 +1874,7 @@ func (s *Syncer) processAccountResponse(res *accountResponse) { return } // Some accounts are incomplete, leave as is for the storage and contract - // task assigners to pick up and fill. + // task assigners to pick up and fill } // processBytecodeResponse integrates an already validated bytecode response @@ -2004,14 +2005,15 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { s.storageBytes += common.StorageSize(len(key) + len(value)) }, } + owner := account // local assignment for stacktrie writer closure tasks = append(tasks, &storageTask{ Next: common.Hash{}, Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { + genTrie: trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) - }, account), + }), }) for r.Next() { batch := ethdb.HookedBatch{ @@ -2025,9 +2027,9 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { Last: r.End(), root: acc.Root, genBatch: batch, - genTrie: trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { + genTrie: trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) - }, account), + }), }) } for _, task := range tasks { @@ -2072,9 +2074,10 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { slots += len(res.hashes[i]) if i < len(res.hashes)-1 || res.subTask == nil { - tr := trie.NewStackTrieWithOwner(func(owner common.Hash, path []byte, hash common.Hash, val []byte) { - rawdb.WriteTrieNode(batch, owner, path, hash, val, s.scheme) - }, account) + // no need to make local reassignment of account: this closure does not outlive the loop + tr := trie.NewStackTrie(func(path []byte, hash common.Hash, val []byte) { + rawdb.WriteTrieNode(batch, account, path, hash, val, s.scheme) + }) for j := 0; j < len(res.hashes[i]); j++ { tr.Update(res.hashes[i][j][:], res.slots[i][j]) } @@ -2394,17 +2397,11 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco for i, key := range hashes { keys[i] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, len(proof)) + nodes := make(trienode.ProofList, len(proof)) for i, node := range proof { nodes[i] = node } - proofdb := nodes.NodeSet() - - var end []byte - if len(keys) > 0 { - end = keys[len(keys)-1] - } - cont, err := trie.VerifyRangeProof(root, req.origin[:], end, keys, accounts, proofdb) + cont, err := trie.VerifyRangeProof(root, req.origin[:], keys, accounts, nodes.Set()) if err != nil { logger.Warn("Account range failed proof", "err", err) // Signal this request as failed, and ready for rescheduling @@ -2621,7 +2618,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // the requested data. For storage range queries that means the state being // retrieved was either already pruned remotely, or the peer is not yet // synced to our head. - if len(hashes) == 0 { + if len(hashes) == 0 && len(proof) == 0 { logger.Debug("Peer rejected storage request") s.statelessPeers[peer.ID()] = struct{}{} s.lock.Unlock() @@ -2633,13 +2630,20 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo // Reconstruct the partial tries from the response and verify them var cont bool + // If a proof was attached while the response is empty, it indicates that the + // requested range specified with 'origin' is empty. Construct an empty state + // response locally to finalize the range. + if len(hashes) == 0 && len(proof) > 0 { + hashes = append(hashes, []common.Hash{}) + slots = append(slots, [][]byte{}) + } for i := 0; i < len(hashes); i++ { // Convert the keys and proofs into an internal format keys := make([][]byte, len(hashes[i])) for j, key := range hashes[i] { keys[j] = common.CopyBytes(key[:]) } - nodes := make(light.NodeList, 0, len(proof)) + nodes := make(trienode.ProofList, 0, len(proof)) if i == len(hashes)-1 { for _, node := range proof { nodes = append(nodes, node) @@ -2649,7 +2653,7 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo if len(nodes) == 0 { // No proof has been attached, the response must cover the entire key // space and hash to the origin root. - _, err = trie.VerifyRangeProof(req.roots[i], nil, nil, keys, slots[i], nil) + _, err = trie.VerifyRangeProof(req.roots[i], nil, keys, slots[i], nil) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage slots failed proof", "err", err) @@ -2658,13 +2662,9 @@ func (s *Syncer) OnStorage(peer SyncPeer, id uint64, hashes [][]common.Hash, slo } else { // A proof was attached, the response is only partial, check that the // returned data is indeed part of the storage trie - proofdb := nodes.NodeSet() + proofdb := nodes.Set() - var end []byte - if len(keys) > 0 { - end = keys[len(keys)-1] - } - cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], end, keys, slots[i], proofdb) + cont, err = trie.VerifyRangeProof(req.roots[i], req.origin[:], keys, slots[i], proofdb) if err != nil { s.scheduleRevertStorageRequest(req) // reschedule request logger.Warn("Storage range failed proof", "err", err) diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 1514ad4e13..5d4099a814 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "fmt" "math/big" + mrand "math/rand" "sync" "testing" "time" @@ -31,10 +32,10 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/testutil" "github.com/ethereum/go-ethereum/trie/triedb/pathdb" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" @@ -254,7 +255,7 @@ func defaultAccountRequestHandler(t *testPeer, id uint64, root common.Hash, orig func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.Hash, limit common.Hash, cap uint64) (keys []common.Hash, vals [][]byte, proofs [][]byte) { var size uint64 if limit == (common.Hash{}) { - limit = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + limit = common.MaxHash } for _, entry := range t.accountValues { if size > cap { @@ -273,7 +274,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H // Unless we send the entire trie, we need to supply proofs // Actually, we need to supply proofs either way! This seems to be an implementation // quirk in go-ethereum - proof := light.NewNodeSet() + proof := trienode.NewProofSet() if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove inexistence of origin", "origin", origin, "error", err) } @@ -283,7 +284,7 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H t.logger.Error("Could not prove last item", "error", err) } } - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } return keys, vals, proofs @@ -319,7 +320,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm if len(origin) > 0 { originHash = common.BytesToHash(origin) } - var limitHash = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + var limitHash = common.MaxHash if len(limit) > 0 { limitHash = common.BytesToHash(limit) } @@ -353,7 +354,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm if originHash != (common.Hash{}) || (abort && len(keys) > 0) { // If we're aborting, we need to prove the first and last item // This terminates the response (and thus the loop) - proof := light.NewNodeSet() + proof := trienode.NewProofSet() stTrie := t.storageTries[account] // Here's a potential gotcha: when constructing the proof, we cannot @@ -368,7 +369,7 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm t.logger.Error("Could not prove last item", "error", err) } } - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } break @@ -411,7 +412,7 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco if exit { // If we're aborting, we need to prove the first and last item // This terminates the response (and thus the loop) - proof := light.NewNodeSet() + proof := trienode.NewProofSet() stTrie := t.storageTries[account] // Here's a potential gotcha: when constructing the proof, we cannot @@ -427,7 +428,7 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco t.logger.Error("Could not prove last item", "error", err) } } - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } break @@ -599,9 +600,10 @@ func testSyncBloatedProof(t *testing.T, scheme string) { vals = append(vals, entry.v) } // The proofs - proof := light.NewNodeSet() + proof := trienode.NewProofSet() if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove origin", "origin", origin, "error", err) + t.logger.Error("Could not prove origin", "origin", origin, "error", err) } // The bloat: add proof of every single element for _, entry := range t.accountValues { @@ -614,7 +616,7 @@ func testSyncBloatedProof(t *testing.T, scheme string) { keys = append(keys[:1], keys[2:]...) vals = append(vals[:1], vals[2:]...) } - for _, blob := range proof.NodeList() { + for _, blob := range proof.List() { proofs = append(proofs, blob) } if err := t.remote.OnAccounts(t, requestId, keys, vals, proofs); err != nil { @@ -762,7 +764,7 @@ func testSyncWithStorage(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 3000, true, false, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -772,7 +774,7 @@ func testSyncWithStorage(t *testing.T, scheme string) { source.storageValues = storageElems return source } - syncer := setupSyncer(nodeScheme, mkSource("sourceA")) + syncer := setupSyncer(scheme, mkSource("sourceA")) done := checkStall(t, term) if err := syncer.Sync(sourceAccountTrie.Hash(), cancel); err != nil { t.Fatalf("sync failed: %v", err) @@ -799,7 +801,7 @@ func testMultiSyncManyUseless(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -821,7 +823,7 @@ func testMultiSyncManyUseless(t *testing.T, scheme string) { } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("full", true, true, true), mkSource("noAccounts", false, true, true), mkSource("noStorage", true, false, true), @@ -853,7 +855,7 @@ func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -875,7 +877,7 @@ func testMultiSyncManyUselessWithLowTimeout(t *testing.T, scheme string) { } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("full", true, true, true), mkSource("noAccounts", false, true, true), mkSource("noStorage", true, false, true), @@ -912,7 +914,7 @@ func testMultiSyncManyUnresponsive(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false, false) mkSource := func(name string, noAccount, noStorage, noTrieNode bool) *testPeer { source := newTestPeer(name, t, term) @@ -934,7 +936,7 @@ func testMultiSyncManyUnresponsive(t *testing.T, scheme string) { } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("full", true, true, true), mkSource("noAccounts", false, true, true), mkSource("noStorage", true, false, true), @@ -1215,7 +1217,7 @@ func testSyncBoundaryStorageTrie(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 10, 1000, false, true, false) mkSource := func(name string) *testPeer { source := newTestPeer(name, t, term) @@ -1226,7 +1228,7 @@ func testSyncBoundaryStorageTrie(t *testing.T, scheme string) { return source } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("peer-a"), mkSource("peer-b"), ) @@ -1257,7 +1259,7 @@ func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 300, 1000, false, false, false) mkSource := func(name string, slow bool) *testPeer { source := newTestPeer(name, t, term) @@ -1273,7 +1275,7 @@ func testSyncWithStorageAndOneCappedPeer(t *testing.T, scheme string) { } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("nice-a", false), mkSource("slow", true), ) @@ -1304,7 +1306,7 @@ func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1317,7 +1319,7 @@ func testSyncWithStorageAndCorruptPeer(t *testing.T, scheme string) { } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("nice-a", defaultStorageRequestHandler), mkSource("nice-b", defaultStorageRequestHandler), mkSource("nice-c", defaultStorageRequestHandler), @@ -1348,7 +1350,7 @@ func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) { }) } ) - nodeScheme, sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false) + sourceAccountTrie, elems, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 100, 3000, true, false, false) mkSource := func(name string, handler storageHandlerFunc) *testPeer { source := newTestPeer(name, t, term) @@ -1360,7 +1362,7 @@ func testSyncWithStorageAndNonProvingPeer(t *testing.T, scheme string) { return source } syncer := setupSyncer( - nodeScheme, + scheme, mkSource("nice-a", defaultStorageRequestHandler), mkSource("nice-b", defaultStorageRequestHandler), mkSource("nice-c", defaultStorageRequestHandler), @@ -1413,6 +1415,45 @@ func testSyncWithStorageMisbehavingProve(t *testing.T, scheme string) { verifyTrie(scheme, syncer.db, sourceAccountTrie.Hash(), t) } +// TestSyncWithUnevenStorage tests sync where the storage trie is not even +// and with a few empty ranges. +func TestSyncWithUnevenStorage(t *testing.T) { + t.Parallel() + + testSyncWithUnevenStorage(t, rawdb.HashScheme) + testSyncWithUnevenStorage(t, rawdb.PathScheme) +} + +func testSyncWithUnevenStorage(t *testing.T, scheme string) { + var ( + once sync.Once + cancel = make(chan struct{}) + term = func() { + once.Do(func() { + close(cancel) + }) + } + ) + accountTrie, accounts, storageTries, storageElems := makeAccountTrieWithStorage(scheme, 3, 256, false, false, true) + + mkSource := func(name string) *testPeer { + source := newTestPeer(name, t, term) + source.accountTrie = accountTrie.Copy() + source.accountValues = accounts + source.setStorageTries(storageTries) + source.storageValues = storageElems + source.storageRequestHandler = func(t *testPeer, reqId uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, max uint64) error { + return defaultStorageRequestHandler(t, reqId, root, accounts, origin, limit, 128) // retrieve storage in large mode + } + return source + } + syncer := setupSyncer(scheme, mkSource("source")) + if err := syncer.Sync(accountTrie.Hash(), cancel); err != nil { + t.Fatalf("sync failed: %v", err) + } + verifyTrie(scheme, syncer.db, accountTrie.Hash(), t) +} + type kv struct { k, v []byte } @@ -1511,7 +1552,7 @@ func makeBoundaryAccountTrie(scheme string, n int) (string, *trie.Trie, []*kv) { for i := 0; i < accountConcurrency; i++ { last := common.BigToHash(new(big.Int).Add(next.Big(), step)) if i == accountConcurrency-1 { - last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + last = common.MaxHash } boundaries = append(boundaries, last) next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) @@ -1608,7 +1649,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(scheme string, accounts, slots } // makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { +func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, boundary bool, uneven bool) (*trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase(), newDbConfig(scheme)) accTrie = trie.NewEmpty(db) @@ -1633,6 +1674,8 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda ) if boundary { stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + } else if uneven { + stRoot, stNodes, stEntries = makeUnevenStorageTrie(common.BytesToHash(key), slots, db) } else { stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) } @@ -1675,7 +1718,7 @@ func makeAccountTrieWithStorage(scheme string, accounts, slots int, code, bounda } storageTries[common.BytesToHash(key)] = trie } - return db.Scheme(), accTrie, entries, storageTries, storageEntries + return accTrie, entries, storageTries, storageEntries } // makeStorageTrieWithSeed fills a storage trie with n items, returning the @@ -1721,7 +1764,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo for i := 0; i < accountConcurrency; i++ { last := common.BigToHash(new(big.Int).Add(next.Big(), step)) if i == accountConcurrency-1 { - last = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + last = common.MaxHash } boundaries = append(boundaries, last) next = common.BigToHash(new(big.Int).Add(last.Big(), common.Big1)) @@ -1752,6 +1795,38 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo return root, nodes, entries } +// makeUnevenStorageTrie constructs a storage tries will states distributed in +// different range unevenly. +func makeUnevenStorageTrie(owner common.Hash, slots int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { + var ( + entries []*kv + tr, _ = trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) + chosen = make(map[byte]struct{}) + ) + for i := 0; i < 3; i++ { + var n int + for { + n = mrand.Intn(15) // the last range is set empty deliberately + if _, ok := chosen[byte(n)]; ok { + continue + } + chosen[byte(n)] = struct{}{} + break + } + for j := 0; j < slots/3; j++ { + key := append([]byte{byte(n)}, testutil.RandBytes(31)...) + val, _ := rlp.EncodeToBytes(testutil.RandBytes(32)) + + elem := &kv{key, val} + tr.MustUpdate(elem.k, elem.v) + entries = append(entries, elem) + } + } + slices.SortFunc(entries, (*kv).cmp) + root, nodes, _ := tr.Commit(false) + return root, nodes, entries +} + func verifyTrie(scheme string, db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Helper() triedb := trie.NewDatabase(rawdb.NewDatabase(db), newDbConfig(scheme)) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 5aa00aad4e..691aa73299 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build (arm64 || amd64) && !openbsd - // Package pebble implements the key-value database layer based on pebble. package pebble @@ -71,6 +69,8 @@ type Database struct { seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt manualMemAllocGauge metrics.Gauge // Gauge for tracking amount of non-managed memory currently allocated + levelsGauge []metrics.Gauge // Gauge for tracking the number of tables in levels + quitLock sync.RWMutex // Mutex protecting the quit channel and the closed flag quitChan chan chan error // Quit channel to stop the metrics collection before closing the database closed bool // keep track of whether we're Closed @@ -146,8 +146,15 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e // The max memtable size is limited by the uint32 offsets stored in // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. - // Taken from https://github.com/cockroachdb/pebble/blob/master/open.go#L38 - maxMemTableSize := 4<<30 - 1 // Capped by 4 GB + // + // - MaxUint32 on 64-bit platforms; + // - MaxInt on 32-bit platforms. + // + // It is used when slices are limited to Uint32 on 64-bit platforms (the + // length limit for slices is naturally MaxInt on 32-bit platforms). + // + // Taken from https://github.com/cockroachdb/pebble/blob/master/internal/constants/constants.go + maxMemTableSize := (1<<31)<<(^uint(0)>>63) - 1 // Two memory tables is configured which is identical to leveldb, // including a frozen memory table and another live one. @@ -230,7 +237,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool, e db.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) // Start up the metrics gathering and return - go db.meter(metricsGatheringInterval) + go db.meter(metricsGatheringInterval, namespace) return db, nil } @@ -427,7 +434,7 @@ func (d *Database) Path() string { // meter periodically retrieves internal pebble counters and reports them to // the metrics subsystem. -func (d *Database) meter(refresh time.Duration) { +func (d *Database) meter(refresh time.Duration, namespace string) { var errc chan error timer := time.NewTimer(refresh) defer timer.Stop() @@ -450,7 +457,7 @@ func (d *Database) meter(refresh time.Duration) { compRead int64 nWrite int64 - metrics = d.db.Metrics() + stats = d.db.Metrics() compTime = d.compTime.Load() writeDelayCount = d.writeDelayCount.Load() writeDelayTime = d.writeDelayTime.Load() @@ -461,14 +468,14 @@ func (d *Database) meter(refresh time.Duration) { writeDelayCounts[i%2] = writeDelayCount compTimes[i%2] = compTime - for _, levelMetrics := range metrics.Levels { + for _, levelMetrics := range stats.Levels { nWrite += int64(levelMetrics.BytesCompacted) nWrite += int64(levelMetrics.BytesFlushed) compWrite += int64(levelMetrics.BytesCompacted) compRead += int64(levelMetrics.BytesRead) } - nWrite += int64(metrics.WAL.BytesWritten) + nWrite += int64(stats.WAL.BytesWritten) compWrites[i%2] = compWrite compReads[i%2] = compRead @@ -490,7 +497,7 @@ func (d *Database) meter(refresh time.Duration) { d.compWriteMeter.Mark(compWrites[i%2] - compWrites[(i-1)%2]) } if d.diskSizeGauge != nil { - d.diskSizeGauge.Update(int64(metrics.DiskSpaceUsage())) + d.diskSizeGauge.Update(int64(stats.DiskSpaceUsage())) } if d.diskReadMeter != nil { d.diskReadMeter.Mark(0) // pebble doesn't track non-compaction reads @@ -499,12 +506,20 @@ func (d *Database) meter(refresh time.Duration) { d.diskWriteMeter.Mark(nWrites[i%2] - nWrites[(i-1)%2]) } // See https://github.com/cockroachdb/pebble/pull/1628#pullrequestreview-1026664054 - manuallyAllocated := metrics.BlockCache.Size + int64(metrics.MemTable.Size) + int64(metrics.MemTable.ZombieSize) + manuallyAllocated := stats.BlockCache.Size + int64(stats.MemTable.Size) + int64(stats.MemTable.ZombieSize) d.manualMemAllocGauge.Update(manuallyAllocated) - d.memCompGauge.Update(metrics.Flush.Count) + d.memCompGauge.Update(stats.Flush.Count) d.nonlevel0CompGauge.Update(nonLevel0CompCount) d.level0CompGauge.Update(level0CompCount) - d.seekCompGauge.Update(metrics.Compact.ReadCount) + d.seekCompGauge.Update(stats.Compact.ReadCount) + + for i, level := range stats.Levels { + // Append metrics for additional layers + if i >= len(d.levelsGauge) { + d.levelsGauge = append(d.levelsGauge, metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil)) + } + d.levelsGauge[i].Update(level.NumFiles) + } // Sleep a bit, then repeat the stats collection select { diff --git a/ethdb/pebble/pebble_test.go b/ethdb/pebble/pebble_test.go index 590d5bf035..1d5611f211 100644 --- a/ethdb/pebble/pebble_test.go +++ b/ethdb/pebble/pebble_test.go @@ -14,8 +14,6 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build (arm64 || amd64) && !openbsd - package pebble import ( diff --git a/go.mod b/go.mod index 15832e80c9..9eed6e5c78 100644 --- a/go.mod +++ b/go.mod @@ -3,26 +3,26 @@ module github.com/ethereum/go-ethereum go 1.20 require ( - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/Microsoft/go-winio v0.6.1 - github.com/VictoriaMetrics/fastcache v1.6.0 - github.com/aws/aws-sdk-go-v2 v1.2.0 - github.com/aws/aws-sdk-go-v2/config v1.1.1 - github.com/aws/aws-sdk-go-v2/credentials v1.1.1 - github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 + github.com/VictoriaMetrics/fastcache v1.12.1 + github.com/aws/aws-sdk-go-v2 v1.21.2 + github.com/aws/aws-sdk-go-v2/config v1.18.45 + github.com/aws/aws-sdk-go-v2/credentials v1.13.43 + github.com/aws/aws-sdk-go-v2/service/route53 v1.30.2 github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 - github.com/cloudflare/cloudflare-go v0.14.0 + github.com/cloudflare/cloudflare-go v0.79.0 github.com/cockroachdb/errors v1.8.1 github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 - github.com/consensys/gnark-crypto v0.10.0 + github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-kzg-4844 v0.3.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v24.0.5+incompatible github.com/dop251/goja v0.0.0-20230806174421-c933cf95e127 github.com/ethereum/c-kzg-4844 v0.3.1 - github.com/fatih/color v1.7.0 + github.com/fatih/color v1.13.0 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/fsnotify/fsnotify v1.6.0 @@ -30,7 +30,7 @@ require ( github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b github.com/go-stack/stack v1.8.1 github.com/gofrs/flock v0.8.1 - github.com/golang-jwt/jwt/v4 v4.3.0 + github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa @@ -58,16 +58,16 @@ require ( github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 github.com/supranational/blst v0.3.11 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.12.0 + golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.13.0 @@ -76,17 +76,21 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/DataDog/zstd v1.4.5 // indirect github.com/StackExchange/wmi v1.2.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 // indirect - github.com/aws/smithy-go v1.1.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect + github.com/aws/smithy-go v1.15.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.5.0 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect @@ -101,15 +105,20 @@ require ( github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect github.com/go-ole/go-ole v1.2.5 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect github.com/klauspost/compress v1.15.15 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect @@ -122,13 +131,14 @@ require ( github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.15.0 // indirect + golang.org/x/net v0.17.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools/v3 v3.5.1 // indirect diff --git a/go.sum b/go.sum index dfbe5420bc..44b705336a 100644 --- a/go.sum +++ b/go.sum @@ -32,12 +32,15 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 h1:8q4SaHjFsClSvuVne0ID/5Ka8u3fcIHyqkLjcFpNRHQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -52,8 +55,8 @@ github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= -github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -63,31 +66,39 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aws/aws-sdk-go-v2 v1.2.0 h1:BS+UYpbsElC82gB+2E2jiCBg36i8HlubTB/dO/moQ9c= -github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= -github.com/aws/aws-sdk-go-v2/config v1.1.1 h1:ZAoq32boMzcaTW9bcUacBswAmHTbvlvDJICgHFZuECo= -github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= -github.com/aws/aws-sdk-go-v2/credentials v1.1.1 h1:NbvWIM1Mx6sNPTxowHgS2ewXCRp+NGTzUYb/96FZJbY= -github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2 h1:EtEU7WRaWliitZh2nmuxEXrN0Cb8EgPUFGIoTMeqbzI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2 h1:4AH9fFjUlVktQMznF+YN33aWNXaR4VgDXyP28qokJC0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= -github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1 h1:cKr6St+CtC3/dl/rEBJvlk7A/IN5D5F02GNkGzfbtVU= -github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= -github.com/aws/aws-sdk-go-v2/service/sso v1.1.1 h1:37QubsarExl5ZuCBlnRP+7l1tNwZPBSTqpTBrPH98RU= -github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= -github.com/aws/aws-sdk-go-v2/service/sts v1.1.1 h1:TJoIfnIFubCX0ACVeJ0w46HEH5MwjwYN4iFhuYIhfIY= -github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= -github.com/aws/smithy-go v1.1.0 h1:D6CSsM3gdxaGaqXnPgOBCeL6Mophqzu7KJOu7zW78sU= -github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= +github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA= +github.com/aws/aws-sdk-go-v2 v1.21.2/go.mod h1:ErQhvNuEMhJjweavOYhxVkn2RUx7kQXVATHrjKtxIpM= +github.com/aws/aws-sdk-go-v2/config v1.18.45 h1:Aka9bI7n8ysuwPeFdm77nfbyHCAKQ3z9ghB3S/38zes= +github.com/aws/aws-sdk-go-v2/config v1.18.45/go.mod h1:ZwDUgFnQgsazQTnWfeLWk5GjeqTQTL8lMkoE1UXzxdE= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43 h1:LU8vo40zBlo3R7bAvBVy/ku4nxGEyZe9N8MqAeFTzF8= +github.com/aws/aws-sdk-go-v2/credentials v1.13.43/go.mod h1:zWJBz1Yf1ZtX5NGax9ZdNjhhI4rgjfgsyk6vTY1yfVg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 h1:PIktER+hwIG286DqXyvVENjgLTAwGgoeriLDD5C+YlQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13/go.mod h1:f/Ib/qYjhV2/qdsf79H3QP/eRE4AkVyEf6sk7XfZ1tg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43 h1:nFBQlGtkbPzp/NjZLuFxRqmT91rLJkgvsEQs68h962Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.43/go.mod h1:auo+PiyLl0n1l8A0e8RIeR8tOzYPfZZH/JNlrJ8igTQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37 h1:JRVhO25+r3ar2mKGP7E0LDl8K9/G36gjlqca5iQbaqc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.37/go.mod h1:Qe+2KtKml+FEsQF/DHmDV+xjtche/hwoF75EG4UlHW8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45 h1:hze8YsjSh8Wl1rYa1CJpRmXP21BvOBuc76YhW0HsuQ4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.45/go.mod h1:lD5M20o09/LCuQ2mE62Mb/iSdSlCNuj6H5ci7tW7OsE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37 h1:WWZA/I2K4ptBS1kg0kV1JbBtG/umed0vwHRrmcr9z7k= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.37/go.mod h1:vBmDnwWXWxNPFRMmG2m/3MKOe+xEcMDo1tanpaWCcck= +github.com/aws/aws-sdk-go-v2/service/route53 v1.30.2 h1:/RPQNjh1sDIezpXaFIkZb7MlXnSyAqjVdAwcJuGYTqg= +github.com/aws/aws-sdk-go-v2/service/route53 v1.30.2/go.mod h1:TQZBt/WaQy+zTHoW++rnl8JBrmZ0VO6EUbVua1+foCA= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2 h1:JuPGc7IkOP4AaqcZSIcyqLpFSqBWK32rM9+a1g6u73k= +github.com/aws/aws-sdk-go-v2/service/sso v1.15.2/go.mod h1:gsL4keucRCgW+xA85ALBpRFfdSLH4kHOVSnLMSuBECo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3 h1:HFiiRkf1SdaAmV3/BHOFZ9DjFynPHj8G/UIO1lQS+fk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.17.3/go.mod h1:a7bHA82fyUXOm+ZSWKU6PIoBxrjSprdLoM8xPYvzYVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 h1:0BkLfgeDjfZnZ+MhB3ONb01u9pwFYTCZVhlsSSBvlbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.23.2/go.mod h1:Eows6e1uQEsc4ZaHANmsPRzAKcVDrcmjjWiih2+HUUQ= +github.com/aws/smithy-go v1.15.0 h1:PS/durmlzvAFpQHDs4wi4sNNP9ExsqZh6IlfdHXgKK8= +github.com/aws/smithy-go v1.15.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= -github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -105,8 +116,8 @@ github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86c github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.14.0 h1:gFqGlGl/5f9UGXAaKapCGUfaTCgRKKnzu2VvzMZlOFA= -github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/cloudflare/cloudflare-go v0.79.0 h1:ErwCYDjFCYppDJlDJ/5WhsSmzegAUe2+K9qgFyQDg3M= +github.com/cloudflare/cloudflare-go v0.79.0/go.mod h1:gkHQf9xEubaQPEuerBuoinR9P8bf8a05Lq0X6WKy1Oc= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= @@ -126,13 +137,12 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1: github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= -github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0= @@ -158,9 +168,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY= github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= @@ -178,8 +186,8 @@ github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHj github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e h1:bBLctRc7kr01YGvaDfgLbTwjFNW5jdp5y5rj8XXBHfY= github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e/go.mod h1:AzA8Lj6YtixmJWL+wkKoBGsLWy9gFrAzi4g+5bCKwpY= @@ -227,6 +235,8 @@ github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -236,8 +246,8 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= -github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= -github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -267,7 +277,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -282,11 +291,15 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa h1:Q75Upo5UN4JbPFURXZ8nLKYUvF85dyFRop/vQ0Rv+64= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -316,6 +329,12 @@ github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY4 github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -348,7 +367,9 @@ github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7Bd github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267 h1:TMtDYDHKYY15rFihtRfck/bfFqNfvcabqvXAFQfAUpY= github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267/go.mod h1:h1nSAbGFqGVzn6Jyl1R/iCcBUHN4g+gW1u9CoBTrb9E= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -386,8 +407,9 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -406,18 +428,21 @@ github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIG github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= @@ -439,7 +464,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -469,6 +493,8 @@ github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQm github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -501,13 +527,15 @@ github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7 h1:cZC+usqsYgHtlBaGulVnZ1hfKAi8iWtujBnRLQE698c= github.com/protolambda/bls12-381-util v0.0.0-20220416220906-d8552aa452c7/go.mod h1:IToEjHuttnUzwZI5KBSM/LOOW3qLbbrHOEfp3SbECGY= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -531,17 +559,13 @@ github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobt github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -554,7 +578,6 @@ github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2n github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= @@ -596,8 +619,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -667,17 +690,13 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -748,10 +767,10 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -759,10 +778,11 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -771,7 +791,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -935,7 +954,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkep gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -943,7 +961,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= diff --git a/internal/build/azure.go b/internal/build/azure.go index 9d1c4f300a..4085228d14 100644 --- a/internal/build/azure.go +++ b/internal/build/azure.go @@ -22,6 +22,7 @@ import ( "os" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" ) // AzureBlobstoreConfig is an authentication and configuration struct containing @@ -48,8 +49,8 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) if err != nil { return err } - u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) - container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + a := fmt.Sprintf("https://%s.blob.core.windows.net/", config.Account) + client, err := azblob.NewClientWithSharedKeyCredential(a, credential, nil) if err != nil { return err } @@ -60,38 +61,38 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig) } defer in.Close() - blockblob := container.NewBlockBlobClient(name) - _, err = blockblob.Upload(context.Background(), in, nil) + _, err = client.UploadFile(context.Background(), config.Container, name, in, nil) return err } // AzureBlobstoreList lists all the files contained within an azure blobstore. -func AzureBlobstoreList(config AzureBlobstoreConfig) ([]*azblob.BlobItemInternal, error) { +func AzureBlobstoreList(config AzureBlobstoreConfig) ([]*container.BlobItem, error) { // Create an authenticated client against the Azure cloud credential, err := azblob.NewSharedKeyCredential(config.Account, config.Token) if err != nil { return nil, err } - u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) - container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + a := fmt.Sprintf("https://%s.blob.core.windows.net/", config.Account) + client, err := azblob.NewClientWithSharedKeyCredential(a, credential, nil) if err != nil { return nil, err } - var maxResults int32 = 5000 - pager := container.ListBlobsFlat(&azblob.ContainerListBlobFlatSegmentOptions{ - Maxresults: &maxResults, - }) - var allBlobs []*azblob.BlobItemInternal - for pager.NextPage(context.Background()) { - res := pager.PageResponse() - allBlobs = append(allBlobs, res.ContainerListBlobFlatSegmentResult.Segment.BlobItems...) + pager := client.NewListBlobsFlatPager(config.Container, nil) + + var blobs []*container.BlobItem + for pager.More() { + page, err := pager.NextPage(context.TODO()) + if err != nil { + return nil, err + } + blobs = append(blobs, page.Segment.BlobItems...) } - return allBlobs, pager.Err() + return blobs, nil } // AzureBlobstoreDelete iterates over a list of files to delete and removes them // from the blobstore. -func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []*azblob.BlobItemInternal) error { +func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []*container.BlobItem) error { if *DryRunFlag { for _, blob := range blobs { fmt.Printf("would delete %s (%s) from %s/%s\n", *blob.Name, blob.Properties.LastModified, config.Account, config.Container) @@ -103,15 +104,14 @@ func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []*azblob.BlobItemI if err != nil { return err } - u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container) - container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil) + a := fmt.Sprintf("https://%s.blob.core.windows.net/", config.Account) + client, err := azblob.NewClientWithSharedKeyCredential(a, credential, nil) if err != nil { return err } // Iterate over the blobs and delete them for _, blob := range blobs { - blockblob := container.NewBlockBlobClient(*blob.Name) - if _, err := blockblob.Delete(context.Background(), &azblob.DeleteBlobOptions{}); err != nil { + if _, err := client.DeleteBlob(context.Background(), config.Container, *blob.Name, nil); err != nil { return err } fmt.Printf("deleted %s (%s)\n", *blob.Name, blob.Properties.LastModified) diff --git a/internal/build/gotool.go b/internal/build/gotool.go index 296ba8c36e..32ca20e869 100644 --- a/internal/build/gotool.go +++ b/internal/build/gotool.go @@ -84,7 +84,11 @@ func (g *GoToolchain) goTool(command string, args ...string) *exec.Cmd { // DownloadGo downloads the Go binary distribution and unpacks it into a temporary // directory. It returns the GOROOT of the unpacked toolchain. -func DownloadGo(csdb *ChecksumDB, version string) string { +func DownloadGo(csdb *ChecksumDB) string { + version, err := Version(csdb, "golang") + if err != nil { + log.Fatal(err) + } // Shortcut: if the Go version that runs this script matches the // requested version exactly, there is no need to download anything. activeGo := strings.TrimPrefix(runtime.Version(), "go") @@ -126,3 +130,52 @@ func DownloadGo(csdb *ChecksumDB, version string) string { } return goroot } + +// Version returns the versions defined in the checksumdb. +func Version(csdb *ChecksumDB, version string) (string, error) { + for _, l := range csdb.allChecksums { + if !strings.HasPrefix(l, "# version:") { + continue + } + v := strings.Split(l, ":")[1] + parts := strings.Split(v, " ") + if len(parts) != 2 { + log.Print("Erroneous version-string", "v", l) + continue + } + if parts[0] == version { + log.Printf("Found version %q", parts[1]) + return parts[1], nil + } + } + return "", fmt.Errorf("no version found for '%v'", version) +} + +// DownloadAndVerifyChecksums downloads all files and checks that they match +// the checksum given in checksums.txt. +// This task can be used to sanity-check new checksums. +func DownloadAndVerifyChecksums(csdb *ChecksumDB) { + var ( + base = "" + ucache = os.TempDir() + ) + for _, l := range csdb.allChecksums { + if strings.HasPrefix(l, "# https://") { + base = l[2:] + continue + } + if strings.HasPrefix(l, "#") { + continue + } + hashFile := strings.Split(l, " ") + if len(hashFile) != 2 { + continue + } + file := hashFile[1] + url := base + file + dst := filepath.Join(ucache, file) + if err := csdb.DownloadFile(url, dst); err != nil { + log.Print(err) + } + } +} diff --git a/les/client_handler.go b/les/client_handler.go index 4cfeba08fe..50f6dce879 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/trie/trienode" ) // clientHandler is responsible for receiving and processing all incoming server @@ -236,7 +237,7 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { p.Log().Trace("Received les/2 proofs response") var resp struct { ReqID, BV uint64 - Data light.NodeList + Data trienode.ProofList } if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) diff --git a/les/handler_test.go b/les/handler_test.go index 26a083f475..c803a5ddb3 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" ) func expectResponse(r p2p.MsgReader, msgcode, reqID, bv uint64, data interface{}) error { @@ -401,7 +402,7 @@ func testGetProofs(t *testing.T, protocol int) { bc := server.handler.blockchain var proofreqs []ProofReq - proofsV2 := light.NewNodeSet() + proofsV2 := trienode.NewProofSet() accounts := []common.Address{bankAddr, userAddr1, userAddr2, signerAddr, {}} for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ { @@ -419,7 +420,7 @@ func testGetProofs(t *testing.T, protocol int) { } // Send the proof request and verify the response sendRequest(rawPeer.app, GetProofsV2Msg, 42, proofreqs) - if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.NodeList()); err != nil { + if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, proofsV2.List()); err != nil { t.Errorf("proofs mismatch: %v", err) } } @@ -456,10 +457,10 @@ func testGetStaleProof(t *testing.T, protocol int) { var expected []rlp.RawValue if wantOK { - proofsV2 := light.NewNodeSet() + proofsV2 := trienode.NewProofSet() t, _ := trie.New(trie.StateTrieID(header.Root), server.backend.Blockchain().TrieDB()) t.Prove(account, proofsV2) - expected = proofsV2.NodeList() + expected = proofsV2.List() } if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, expected); err != nil { t.Errorf("codes mismatch: %v", err) diff --git a/les/odr_requests.go b/les/odr_requests.go index 2b23e0540c..c907018590 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" ) var ( @@ -222,9 +223,9 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { if msg.MsgType != MsgProofsV2 { return errInvalidMessageType } - proofs := msg.Obj.(light.NodeList) + proofs := msg.Obj.(trienode.ProofList) // Verify the proof and store if checks out - nodeSet := proofs.NodeSet() + nodeSet := proofs.Set() reads := &readTraceDB{db: nodeSet} if _, err := trie.VerifyProof(r.Id.Root, r.Key, reads); err != nil { return fmt.Errorf("merkle proof verification failed: %v", err) @@ -308,7 +309,7 @@ type HelperTrieReq struct { } type HelperTrieResps struct { // describes all responses, not just a single one - Proofs light.NodeList + Proofs trienode.ProofList AuxData [][]byte } @@ -356,7 +357,7 @@ func (r *ChtRequest) Validate(db ethdb.Database, msg *Msg) error { if len(resp.AuxData) != 1 { return errInvalidEntryCount } - nodeSet := resp.Proofs.NodeSet() + nodeSet := resp.Proofs.Set() headerEnc := resp.AuxData[0] if len(headerEnc) == 0 { return errHeaderUnavailable @@ -451,7 +452,7 @@ func (r *BloomRequest) Validate(db ethdb.Database, msg *Msg) error { } resps := msg.Obj.(HelperTrieResps) proofs := resps.Proofs - nodeSet := proofs.NodeSet() + nodeSet := proofs.Set() reads := &readTraceDB{db: nodeSet} r.BloomBits = make([][]byte, len(r.SectionIndexList)) diff --git a/les/peer.go b/les/peer.go index 48381689ef..58cb928700 100644 --- a/les/peer.go +++ b/les/peer.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" ) var ( @@ -899,7 +900,7 @@ func (p *clientPeer) replyReceiptsRLP(reqID uint64, receipts []rlp.RawValue) *re } // replyProofsV2 creates a reply with a batch of merkle proofs, corresponding to the ones requested. -func (p *clientPeer) replyProofsV2(reqID uint64, proofs light.NodeList) *reply { +func (p *clientPeer) replyProofsV2(reqID uint64, proofs trienode.ProofList) *reply { data, _ := rlp.EncodeToBytes(proofs) return &reply{p.rw, ProofsV2Msg, reqID, data} } diff --git a/les/server_requests.go b/les/server_requests.go index 485be6d9e9..9a249f04c9 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" ) // serverBackend defines the backend functions needed for serving LES requests @@ -378,7 +379,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { err error ) bc := backend.BlockChain() - nodes := light.NewNodeSet() + nodes := trienode.NewProofSet() for i, request := range r.Reqs { if i != 0 && !waitOrStop() { @@ -444,7 +445,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { break } } - return p.replyProofsV2(r.ReqID, nodes.NodeList()) + return p.replyProofsV2(r.ReqID, nodes.List()) }, r.ReqID, uint64(len(r.Reqs)), nil } @@ -463,7 +464,7 @@ func handleGetHelperTrieProofs(msg Decoder) (serveRequestFn, uint64, uint64, err auxData [][]byte ) bc := backend.BlockChain() - nodes := light.NewNodeSet() + nodes := trienode.NewProofSet() for i, request := range r.Reqs { if i != 0 && !waitOrStop() { return nil @@ -498,7 +499,7 @@ func handleGetHelperTrieProofs(msg Decoder) (serveRequestFn, uint64, uint64, err break } } - return p.replyHelperTrieProofs(r.ReqID, HelperTrieResps{Proofs: nodes.NodeList(), AuxData: auxData}) + return p.replyHelperTrieProofs(r.ReqID, HelperTrieResps{Proofs: nodes.List(), AuxData: auxData}) }, r.ReqID, uint64(len(r.Reqs)), nil } diff --git a/light/odr.go b/light/odr.go index 2597027435..39f626ee2c 100644 --- a/light/odr.go +++ b/light/odr.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie/trienode" ) // NoOdr is the default context passed to an ODR capable function when the ODR @@ -90,7 +91,7 @@ func StorageTrieID(state *TrieID, address common.Address, root common.Hash) *Tri type TrieRequest struct { Id *TrieID Key []byte - Proof *NodeSet + Proof *trienode.ProofSet } // StoreResult stores the retrieved data in local database @@ -143,7 +144,7 @@ type ChtRequest struct { ChtRoot common.Hash Header *types.Header Td *big.Int - Proof *NodeSet + Proof *trienode.ProofSet } // StoreResult stores the retrieved data in local database @@ -163,7 +164,7 @@ type BloomRequest struct { SectionIndexList []uint64 BloomTrieRoot common.Hash BloomBits [][]byte - Proofs *NodeSet + Proofs *trienode.ProofSet } // StoreResult stores the retrieved data in local database diff --git a/light/odr_test.go b/light/odr_test.go index d8a7f10675..c415d73e7e 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" ) var ( @@ -95,7 +96,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { if err != nil { panic(err) } - nodes := NewNodeSet() + nodes := trienode.NewProofSet() t.Prove(req.Key, nodes) req.Proof = nodes case *CodeRequest: diff --git a/light/postprocess.go b/light/postprocess.go index 13d75f8617..a317e30b90 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -363,7 +363,7 @@ func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uin func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { indexCh := make(chan uint, types.BloomBitLength) type res struct { - nodes *NodeSet + nodes *trienode.ProofSet err error } resCh := make(chan res, types.BloomBitLength) diff --git a/params/config.go b/params/config.go index ac55d3771a..88ff772a1d 100644 --- a/params/config.go +++ b/params/config.go @@ -214,7 +214,7 @@ var ( } // TestChainConfig contains every protocol change (EIPs) introduced - // and accepted by the Ethereum core developers for testing proposes. + // and accepted by the Ethereum core developers for testing purposes. TestChainConfig = &ChainConfig{ ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), diff --git a/params/version.go b/params/version.go index bcffd292a5..f8e77266da 100644 --- a/params/version.go +++ b/params/version.go @@ -23,7 +23,7 @@ import ( const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 13 // Minor version component of the current release - VersionPatch = 3 // Patch version component of the current release + VersionPatch = 4 // Patch version component of the current release VersionMeta = "unstable" // Version metadata to append to the version string ) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index ba490b761f..c9d7815536 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -128,7 +128,7 @@ func (f *fuzzer) fuzz() int { if len(keys) == 0 { return 0 } - var first, last = keys[0], keys[len(keys)-1] + var first = keys[0] testcase %= 6 switch testcase { case 0: @@ -165,7 +165,7 @@ func (f *fuzzer) fuzz() int { } ok = 1 //nodes, subtrie - hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, last, keys, vals, proof) + hasMore, err := trie.VerifyRangeProof(tr.Hash(), first, keys, vals, proof) if err != nil { if hasMore { panic("err != nil && hasMore == true") diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 3d65524095..20b8ca24b3 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -140,8 +140,8 @@ func (f *fuzzer) fuzz() int { trieA = trie.NewEmpty(dbA) spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()} dbB = trie.NewDatabase(rawdb.NewDatabase(spongeB), nil) - trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) + trieB = trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(spongeB, common.Hash{}, path, hash, blob, dbB.Scheme()) }) vals []kv useful bool @@ -205,13 +205,10 @@ func (f *fuzzer) fuzz() int { // Ensure all the nodes are persisted correctly var ( nodeset = make(map[string][]byte) // path -> blob - trieC = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { + trieC = trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { if crypto.Keccak256Hash(blob) != hash { panic("invalid node blob") } - if owner != (common.Hash{}) { - panic("invalid node owner") - } nodeset[string(path)] = common.CopyBytes(blob) }) checked int diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index d1d6fdc665..8b501645b6 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -83,6 +83,7 @@ func Fuzz(input []byte) int { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, + nil, clock, rand, ) f.Start() @@ -116,6 +117,8 @@ func Fuzz(input []byte) int { var ( announceIdxs = make([]int, announce) announces = make([]common.Hash, announce) + types = make([]byte, announce) + sizes = make([]uint32, announce) ) for i := 0; i < len(announces); i++ { annBuf := make([]byte, 2) @@ -124,11 +127,13 @@ func Fuzz(input []byte) int { } announceIdxs[i] = (int(annBuf[0])*256 + int(annBuf[1])) % len(txs) announces[i] = txs[announceIdxs[i]].Hash() + types[i] = txs[announceIdxs[i]].Type() + sizes[i] = uint32(txs[announceIdxs[i]].Size()) } if verbose { fmt.Println("Notify", peer, announceIdxs) } - if err := f.Notify(peer, announces); err != nil { + if err := f.Notify(peer, types, sizes, announces); err != nil { panic(err) } diff --git a/trie/proof.go b/trie/proof.go index a463c80b48..a526a53402 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -481,7 +481,7 @@ func hasRightElement(node node, key []byte) bool { // Note: This method does not verify that the proof is of minimal form. If the input // proofs are 'bloated' with neighbour leaves or random data, aside from the 'useful' // data, then the proof will still be accepted. -func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (bool, error) { +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (bool, error) { if len(keys) != len(values) { return false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) } @@ -520,6 +520,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } return false, nil } + var lastKey = keys[len(keys)-1] // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { diff --git a/trie/proof_test.go b/trie/proof_test.go index fc2de62649..59ae201cea 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -191,7 +191,7 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -221,19 +221,10 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { if bytes.Compare(first, entries[start].k) > 0 { continue } - // Short circuit if the increased key is same with the next key - last := increaseKey(common.CopyBytes(entries[end-1].k)) - if end != len(entries) && bytes.Equal(last, entries[end].k) { - continue - } - // Short circuit if the increased key is overflow - if bytes.Compare(last, entries[end-1].k) < 0 { - continue - } if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -242,36 +233,15 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), first, keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } } - // Special case, two edge proofs for two edge key. - proof := memorydb.New() - first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() - last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() - if err := trie.Prove(first, proof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) - } - if err := trie.Prove(last, proof); err != nil { - t.Fatalf("Failed to prove the last node %v", err) - } - var k [][]byte - var v [][]byte - for i := 0; i < len(entries); i++ { - k = append(k, entries[i].k) - v = append(v, entries[i].v) - } - _, err := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) - if err != nil { - t.Fatal("Failed to verify whole rang with non-existent edges") - } } // TestRangeProofWithInvalidNonExistentProof tests such scenarios: // - There exists a gap between the first element and the left edge proof -// - There exists a gap between the last element and the right edge proof func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { trie, vals := randomTrie(4096) var entries []*kv @@ -298,29 +268,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) - if err == nil { - t.Fatalf("Expected to detect the error, got nil") - } - - // Case 2 - start, end = 100, 200 - last := increaseKey(common.CopyBytes(entries[end-1].k)) - proof = memorydb.New() - if err := trie.Prove(entries[start].k, proof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) - } - if err := trie.Prove(last, proof); err != nil { - t.Fatalf("Failed to prove the last node %v", err) - } - end = 195 // Capped slice - k = make([][]byte, 0) - v = make([][]byte, 0) - for i := start; i < end; i++ { - k = append(k, entries[i].k) - v = append(v, entries[i].v) - } - _, err = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), first, k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -344,7 +292,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err := VerifyRangeProof(trie.Hash(), entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -359,7 +307,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -374,7 +322,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -389,7 +337,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -408,7 +356,7 @@ func TestOneElementRangeProof(t *testing.T) { if err := tinyTrie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) + _, err = VerifyRangeProof(tinyTrie.Hash(), first, [][]byte{entry.k}, [][]byte{entry.v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -430,7 +378,7 @@ func TestAllElementsProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) + _, err := VerifyRangeProof(trie.Hash(), nil, k, v, nil) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -443,7 +391,7 @@ func TestAllElementsProof(t *testing.T) { if err := trie.Prove(entries[len(entries)-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) + _, err = VerifyRangeProof(trie.Hash(), k[0], k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -451,14 +399,13 @@ func TestAllElementsProof(t *testing.T) { // Even with non-existent edge proofs, it should still work. proof = memorydb.New() first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() - last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, proof); err != nil { + if err := trie.Prove(entries[len(entries)-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + _, err = VerifyRangeProof(trie.Hash(), first, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -491,43 +438,7 @@ func TestSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) - if err != nil { - t.Fatalf("Expected no error, got %v", err) - } - } - } -} - -// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. -func TestReverseSingleSideRangeProof(t *testing.T) { - for i := 0; i < 64; i++ { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) - var entries []*kv - for i := 0; i < 4096; i++ { - value := &kv{randBytes(32), randBytes(20), false} - trie.MustUpdate(value.k, value.v) - entries = append(entries, value) - } - slices.SortFunc(entries, (*kv).cmp) - - var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} - for _, pos := range cases { - proof := memorydb.New() - if err := trie.Prove(entries[pos].k, proof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) - } - last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") - if err := trie.Prove(last.Bytes(), proof); err != nil { - t.Fatalf("Failed to prove the last node %v", err) - } - k := make([][]byte, 0) - v := make([][]byte, 0) - for i := pos; i < len(entries); i++ { - k = append(k, entries[i].k) - v = append(v, entries[i].v) - } - _, err := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) + _, err := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -561,7 +472,7 @@ func TestBadRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - var first, last = keys[0], keys[len(keys)-1] + var first = keys[0] testcase := mrand.Intn(6) var index int switch testcase { @@ -576,7 +487,7 @@ func TestBadRangeProof(t *testing.T) { case 2: // Gapped entry slice index = mrand.Intn(end - start) - if (index == 0 && start < 100) || (index == end-start-1 && end <= 100) { + if (index == 0 && start < 100) || (index == end-start-1) { continue } keys = append(keys[:index], keys[index+1:]...) @@ -599,7 +510,7 @@ func TestBadRangeProof(t *testing.T) { index = mrand.Intn(end - start) vals[index] = nil } - _, err := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), first, keys, vals, proof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -633,7 +544,7 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, proof) if err == nil { t.Fatal("expect error, got nil") } @@ -649,24 +560,22 @@ func TestSameSideProofs(t *testing.T) { slices.SortFunc(entries, (*kv).cmp) pos := 1000 - first := decreaseKey(common.CopyBytes(entries[pos].k)) - first = decreaseKey(first) - last := decreaseKey(common.CopyBytes(entries[pos].k)) + first := common.CopyBytes(entries[0].k) proof := memorydb.New() if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, proof); err != nil { - t.Fatalf("Failed to prove the last node %v", err) + if err := trie.Prove(entries[2000].k, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) } - _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, err := VerifyRangeProof(trie.Hash(), first, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } first = increaseKey(common.CopyBytes(entries[pos].k)) - last = increaseKey(common.CopyBytes(entries[pos].k)) + last := increaseKey(common.CopyBytes(entries[pos].k)) last = increaseKey(last) proof = memorydb.New() @@ -676,7 +585,7 @@ func TestSameSideProofs(t *testing.T) { if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + _, err = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) if err == nil { t.Fatalf("Expected error, got nil") } @@ -703,15 +612,12 @@ func TestHasRightElement(t *testing.T) { {50, 100, true}, {50, len(entries), false}, // No more element expected {len(entries) - 1, len(entries), false}, // Single last element with two existent proofs(point to same key) - {len(entries) - 1, -1, false}, // Single last element with non-existent right proof {0, len(entries), false}, // The whole set with existent left proof {-1, len(entries), false}, // The whole set with non-existent left proof - {-1, -1, false}, // The whole set with non-existent left/right proof } for _, c := range cases { var ( firstKey []byte - lastKey []byte start = c.start end = c.end proof = memorydb.New() @@ -727,16 +633,8 @@ func TestHasRightElement(t *testing.T) { t.Fatalf("Failed to prove the first node %v", err) } } - if c.end == -1 { - lastKey, end = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes(), len(entries) - if err := trie.Prove(lastKey, proof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) - } - } else { - lastKey = entries[c.end-1].k - if err := trie.Prove(entries[c.end-1].k, proof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) - } + if err := trie.Prove(entries[c.end-1].k, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) } k := make([][]byte, 0) v := make([][]byte, 0) @@ -744,7 +642,7 @@ func TestHasRightElement(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) + hasMore, err := VerifyRangeProof(trie.Hash(), firstKey, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -777,7 +675,7 @@ func TestEmptyRangeProof(t *testing.T) { if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) + _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, proof) if c.err && err == nil { t.Fatalf("Expected error, got nil") } @@ -817,7 +715,7 @@ func TestBloatedProof(t *testing.T) { trie.Prove(keys[0], want) trie.Prove(keys[len(keys)-1], want) - if _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof); err != nil { + if _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, proof); err != nil { t.Fatalf("expected bloated proof to succeed, got %v", err) } } @@ -860,7 +758,7 @@ func TestEmptyValueRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, proof) if err == nil { t.Fatalf("Expected failure on noop entry") } @@ -895,7 +793,7 @@ func TestAllElementsEmptyValueRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - _, err := VerifyRangeProof(trie.Hash(), nil, nil, keys, vals, nil) + _, err := VerifyRangeProof(trie.Hash(), nil, keys, vals, nil) if err == nil { t.Fatalf("Expected failure on noop entry") } @@ -1001,7 +899,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, values, proof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } @@ -1028,7 +926,7 @@ func benchmarkVerifyRangeNoProof(b *testing.B, size int) { } b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, nil) + _, err := VerifyRangeProof(trie.Hash(), keys[0], keys, values, nil) if err != nil { b.Fatalf("Expected no error, got %v", err) } @@ -1087,15 +985,14 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { root := trie.Hash() proof := memorydb.New() start := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") - end := common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") if err := trie.Prove(start, proof); err != nil { t.Fatalf("failed to prove start: %v", err) } - if err := trie.Prove(end, proof); err != nil { + if err := trie.Prove(keys[len(keys)-1], proof); err != nil { t.Fatalf("failed to prove end: %v", err) } - more, err := VerifyRangeProof(root, start, end, keys, vals, proof) + more, err := VerifyRangeProof(root, start, keys, vals, proof) if err != nil { t.Fatalf("failed to verify range proof: %v", err) } diff --git a/trie/stacktrie.go b/trie/stacktrie.go index 0d65ee75e0..35208e1cb3 100644 --- a/trie/stacktrie.go +++ b/trie/stacktrie.go @@ -17,11 +17,7 @@ package trie import ( - "bufio" - "bytes" - "encoding/gob" "errors" - "io" "sync" "github.com/ethereum/go-ethereum/common" @@ -29,171 +25,87 @@ import ( "github.com/ethereum/go-ethereum/log" ) -var ErrCommitDisabled = errors.New("no database for committing") - -var stPool = sync.Pool{ - New: func() interface{} { - return NewStackTrie(nil) - }, -} +var ( + ErrCommitDisabled = errors.New("no database for committing") + stPool = sync.Pool{New: func() any { return new(stNode) }} + _ = types.TrieHasher((*StackTrie)(nil)) +) // NodeWriteFunc is used to provide all information of a dirty node for committing // so that callers can flush nodes into database with desired scheme. -type NodeWriteFunc = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) - -func stackTrieFromPool(writeFn NodeWriteFunc, owner common.Hash) *StackTrie { - st := stPool.Get().(*StackTrie) - st.owner = owner - st.writeFn = writeFn - return st -} - -func returnToPool(st *StackTrie) { - st.Reset() - stPool.Put(st) -} +type NodeWriteFunc = func(path []byte, hash common.Hash, blob []byte) // StackTrie is a trie implementation that expects keys to be inserted // in order. Once it determines that a subtree will no longer be inserted // into, it will hash it and free up the memory it uses. type StackTrie struct { - owner common.Hash // the owner of the trie - nodeType uint8 // node type (as in branch, ext, leaf) - val []byte // value contained by this node if it's a leaf - key []byte // key chunk covered by this (leaf|ext) node - children [16]*StackTrie // list of children (for branch and exts) - writeFn NodeWriteFunc // function for committing nodes, can be nil + writeFn NodeWriteFunc // function for committing nodes, can be nil + root *stNode + h *hasher } // NewStackTrie allocates and initializes an empty trie. func NewStackTrie(writeFn NodeWriteFunc) *StackTrie { return &StackTrie{ - nodeType: emptyNode, - writeFn: writeFn, + writeFn: writeFn, + root: stPool.Get().(*stNode), + h: newHasher(false), } } -// NewStackTrieWithOwner allocates and initializes an empty trie, but with -// the additional owner field. -func NewStackTrieWithOwner(writeFn NodeWriteFunc, owner common.Hash) *StackTrie { - return &StackTrie{ - owner: owner, - nodeType: emptyNode, - writeFn: writeFn, - } -} - -// NewFromBinary initialises a serialized stacktrie with the given db. -func NewFromBinary(data []byte, writeFn NodeWriteFunc) (*StackTrie, error) { - var st StackTrie - if err := st.UnmarshalBinary(data); err != nil { - return nil, err - } - // If a database is used, we need to recursively add it to every child - if writeFn != nil { - st.setWriter(writeFn) - } - return &st, nil -} - -// MarshalBinary implements encoding.BinaryMarshaler -func (st *StackTrie) MarshalBinary() (data []byte, err error) { - var ( - b bytes.Buffer - w = bufio.NewWriter(&b) - ) - if err := gob.NewEncoder(w).Encode(struct { - Owner common.Hash - NodeType uint8 - Val []byte - Key []byte - }{ - st.owner, - st.nodeType, - st.val, - st.key, - }); err != nil { - return nil, err - } - for _, child := range st.children { - if child == nil { - w.WriteByte(0) - continue - } - w.WriteByte(1) - if childData, err := child.MarshalBinary(); err != nil { - return nil, err - } else { - w.Write(childData) - } - } - w.Flush() - return b.Bytes(), nil -} - -// UnmarshalBinary implements encoding.BinaryUnmarshaler -func (st *StackTrie) UnmarshalBinary(data []byte) error { - r := bytes.NewReader(data) - return st.unmarshalBinary(r) -} - -func (st *StackTrie) unmarshalBinary(r io.Reader) error { - var dec struct { - Owner common.Hash - NodeType uint8 - Val []byte - Key []byte - } - if err := gob.NewDecoder(r).Decode(&dec); err != nil { - return err - } - st.owner = dec.Owner - st.nodeType = dec.NodeType - st.val = dec.Val - st.key = dec.Key - - var hasChild = make([]byte, 1) - for i := range st.children { - if _, err := r.Read(hasChild); err != nil { - return err - } else if hasChild[0] == 0 { - continue - } - var child StackTrie - if err := child.unmarshalBinary(r); err != nil { - return err - } - st.children[i] = &child +// Update inserts a (key, value) pair into the stack trie. +func (t *StackTrie) Update(key, value []byte) error { + k := keybytesToHex(key) + if len(value) == 0 { + panic("deletion not supported") } + t.insert(t.root, k[:len(k)-1], value, nil) return nil } -func (st *StackTrie) setWriter(writeFn NodeWriteFunc) { - st.writeFn = writeFn - for _, child := range st.children { - if child != nil { - child.setWriter(writeFn) - } +// MustUpdate is a wrapper of Update and will omit any encountered error but +// just print out an error message. +func (t *StackTrie) MustUpdate(key, value []byte) { + if err := t.Update(key, value); err != nil { + log.Error("Unhandled trie error in StackTrie.Update", "err", err) } } -func newLeaf(owner common.Hash, key, val []byte, writeFn NodeWriteFunc) *StackTrie { - st := stackTrieFromPool(writeFn, owner) - st.nodeType = leafNode +func (t *StackTrie) Reset() { + t.writeFn = nil + t.root = stPool.Get().(*stNode) +} + +// stNode represents a node within a StackTrie +type stNode struct { + typ uint8 // node type (as in branch, ext, leaf) + key []byte // key chunk covered by this (leaf|ext) node + val []byte // value contained by this node if it's a leaf + children [16]*stNode // list of children (for branch and exts) +} + +// newLeaf constructs a leaf node with provided node key and value. The key +// will be deep-copied in the function and safe to modify afterwards, but +// value is not. +func newLeaf(key, val []byte) *stNode { + st := stPool.Get().(*stNode) + st.typ = leafNode st.key = append(st.key, key...) st.val = val return st } -func newExt(owner common.Hash, key []byte, child *StackTrie, writeFn NodeWriteFunc) *StackTrie { - st := stackTrieFromPool(writeFn, owner) - st.nodeType = extNode +// newExt constructs an extension node with provided node key and child. The +// key will be deep-copied in the function and safe to modify afterwards. +func newExt(key []byte, child *stNode) *stNode { + st := stPool.Get().(*stNode) + st.typ = extNode st.key = append(st.key, key...) st.children[0] = child return st } -// List all values that StackTrie#nodeType can hold +// List all values that stNode#nodeType can hold const ( emptyNode = iota branchNode @@ -202,59 +114,40 @@ const ( hashedNode ) -// Update inserts a (key, value) pair into the stack trie. -func (st *StackTrie) Update(key, value []byte) error { - k := keybytesToHex(key) - if len(value) == 0 { - panic("deletion not supported") +func (n *stNode) reset() *stNode { + n.key = n.key[:0] + n.val = nil + for i := range n.children { + n.children[i] = nil } - st.insert(k[:len(k)-1], value, nil) - return nil -} - -// MustUpdate is a wrapper of Update and will omit any encountered error but -// just print out an error message. -func (st *StackTrie) MustUpdate(key, value []byte) { - if err := st.Update(key, value); err != nil { - log.Error("Unhandled trie error in StackTrie.Update", "err", err) - } -} - -func (st *StackTrie) Reset() { - st.owner = common.Hash{} - st.writeFn = nil - st.key = st.key[:0] - st.val = nil - for i := range st.children { - st.children[i] = nil - } - st.nodeType = emptyNode + n.typ = emptyNode + return n } // Helper function that, given a full key, determines the index // at which the chunk pointed by st.keyOffset is different from // the same chunk in the full key. -func (st *StackTrie) getDiffIndex(key []byte) int { - for idx, nibble := range st.key { +func (n *stNode) getDiffIndex(key []byte) int { + for idx, nibble := range n.key { if nibble != key[idx] { return idx } } - return len(st.key) + return len(n.key) } // Helper function to that inserts a (key, value) pair into // the trie. -func (st *StackTrie) insert(key, value []byte, prefix []byte) { - switch st.nodeType { +func (t *StackTrie) insert(st *stNode, key, value []byte, prefix []byte) { + switch st.typ { case branchNode: /* Branch */ idx := int(key[0]) // Unresolve elder siblings for i := idx - 1; i >= 0; i-- { if st.children[i] != nil { - if st.children[i].nodeType != hashedNode { - st.children[i].hash(append(prefix, byte(i))) + if st.children[i].typ != hashedNode { + t.hash(st.children[i], append(prefix, byte(i))) } break } @@ -262,9 +155,9 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { // Add new child if st.children[idx] == nil { - st.children[idx] = newLeaf(st.owner, key[1:], value, st.writeFn) + st.children[idx] = newLeaf(key[1:], value) } else { - st.children[idx].insert(key[1:], value, append(prefix, key[0])) + t.insert(st.children[idx], key[1:], value, append(prefix, key[0])) } case extNode: /* Ext */ @@ -279,46 +172,46 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { if diffidx == len(st.key) { // Ext key and key segment are identical, recurse into // the child node. - st.children[0].insert(key[diffidx:], value, append(prefix, key[:diffidx]...)) + t.insert(st.children[0], key[diffidx:], value, append(prefix, key[:diffidx]...)) return } // Save the original part. Depending if the break is // at the extension's last byte or not, create an // intermediate extension or use the extension's child // node directly. - var n *StackTrie + var n *stNode if diffidx < len(st.key)-1 { // Break on the non-last byte, insert an intermediate // extension. The path prefix of the newly-inserted // extension should also contain the different byte. - n = newExt(st.owner, st.key[diffidx+1:], st.children[0], st.writeFn) - n.hash(append(prefix, st.key[:diffidx+1]...)) + n = newExt(st.key[diffidx+1:], st.children[0]) + t.hash(n, append(prefix, st.key[:diffidx+1]...)) } else { // Break on the last byte, no need to insert // an extension node: reuse the current node. // The path prefix of the original part should // still be same. n = st.children[0] - n.hash(append(prefix, st.key...)) + t.hash(n, append(prefix, st.key...)) } - var p *StackTrie + var p *stNode if diffidx == 0 { // the break is on the first byte, so // the current node is converted into // a branch node. st.children[0] = nil p = st - st.nodeType = branchNode + st.typ = branchNode } else { // the common prefix is at least one byte // long, insert a new intermediate branch // node. - st.children[0] = stackTrieFromPool(st.writeFn, st.owner) - st.children[0].nodeType = branchNode + st.children[0] = stPool.Get().(*stNode) + st.children[0].typ = branchNode p = st.children[0] } // Create a leaf for the inserted part - o := newLeaf(st.owner, key[diffidx+1:], value, st.writeFn) + o := newLeaf(key[diffidx+1:], value) // Insert both child leaves where they belong: origIdx := st.key[diffidx] @@ -344,18 +237,18 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { // Check if the split occurs at the first nibble of the // chunk. In that case, no prefix extnode is necessary. // Otherwise, create that - var p *StackTrie + var p *stNode if diffidx == 0 { // Convert current leaf into a branch - st.nodeType = branchNode + st.typ = branchNode p = st st.children[0] = nil } else { // Convert current node into an ext, // and insert a child branch node. - st.nodeType = extNode - st.children[0] = NewStackTrieWithOwner(st.writeFn, st.owner) - st.children[0].nodeType = branchNode + st.typ = extNode + st.children[0] = stPool.Get().(*stNode) + st.children[0].typ = branchNode p = st.children[0] } @@ -363,11 +256,11 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { // value and another containing the new value. The child leaf // is hashed directly in order to free up some memory. origIdx := st.key[diffidx] - p.children[origIdx] = newLeaf(st.owner, st.key[diffidx+1:], st.val, st.writeFn) - p.children[origIdx].hash(append(prefix, st.key[:diffidx+1]...)) + p.children[origIdx] = newLeaf(st.key[diffidx+1:], st.val) + t.hash(p.children[origIdx], append(prefix, st.key[:diffidx+1]...)) newIdx := key[diffidx] - p.children[newIdx] = newLeaf(st.owner, key[diffidx+1:], value, st.writeFn) + p.children[newIdx] = newLeaf(key[diffidx+1:], value) // Finally, cut off the key part that has been passed // over to the children. @@ -375,7 +268,7 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { st.val = nil case emptyNode: /* Empty */ - st.nodeType = leafNode + st.typ = leafNode st.key = key st.val = value @@ -398,25 +291,18 @@ func (st *StackTrie) insert(key, value []byte, prefix []byte) { // - And the 'st.type' will be 'hashedNode' AGAIN // // This method also sets 'st.type' to hashedNode, and clears 'st.key'. -func (st *StackTrie) hash(path []byte) { - h := newHasher(false) - defer returnHasherToPool(h) - - st.hashRec(h, path) -} - -func (st *StackTrie) hashRec(hasher *hasher, path []byte) { +func (t *StackTrie) hash(st *stNode, path []byte) { // The switch below sets this to the RLP-encoding of this node. var encodedNode []byte - switch st.nodeType { + switch st.typ { case hashedNode: return case emptyNode: st.val = types.EmptyRootHash.Bytes() st.key = st.key[:0] - st.nodeType = hashedNode + st.typ = hashedNode return case branchNode: @@ -426,23 +312,21 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) { nodes.Children[i] = nilValueNode continue } - child.hashRec(hasher, append(path, byte(i))) + t.hash(child, append(path, byte(i))) + if len(child.val) < 32 { nodes.Children[i] = rawNode(child.val) } else { nodes.Children[i] = hashNode(child.val) } - - // Release child back to pool. st.children[i] = nil - returnToPool(child) + stPool.Put(child.reset()) // Release child back to pool. } - - nodes.encode(hasher.encbuf) - encodedNode = hasher.encodedBytes() + nodes.encode(t.h.encbuf) + encodedNode = t.h.encodedBytes() case extNode: - st.children[0].hashRec(hasher, append(path, st.key...)) + t.hash(st.children[0], append(path, st.key...)) n := shortNode{Key: hexToCompactInPlace(st.key)} if len(st.children[0].val) < 32 { @@ -450,26 +334,24 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) { } else { n.Val = hashNode(st.children[0].val) } + n.encode(t.h.encbuf) + encodedNode = t.h.encodedBytes() - n.encode(hasher.encbuf) - encodedNode = hasher.encodedBytes() - - // Release child back to pool. - returnToPool(st.children[0]) + stPool.Put(st.children[0].reset()) // Release child back to pool. st.children[0] = nil case leafNode: st.key = append(st.key, byte(16)) n := shortNode{Key: hexToCompactInPlace(st.key), Val: valueNode(st.val)} - n.encode(hasher.encbuf) - encodedNode = hasher.encodedBytes() + n.encode(t.h.encbuf) + encodedNode = t.h.encodedBytes() default: panic("invalid node type") } - st.nodeType = hashedNode + st.typ = hashedNode st.key = st.key[:0] if len(encodedNode) < 32 { st.val = common.CopyBytes(encodedNode) @@ -478,18 +360,16 @@ func (st *StackTrie) hashRec(hasher *hasher, path []byte) { // Write the hash to the 'val'. We allocate a new val here to not mutate // input values - st.val = hasher.hashData(encodedNode) - if st.writeFn != nil { - st.writeFn(st.owner, path, common.BytesToHash(st.val), encodedNode) + st.val = t.h.hashData(encodedNode) + if t.writeFn != nil { + t.writeFn(path, common.BytesToHash(st.val), encodedNode) } } // Hash returns the hash of the current node. -func (st *StackTrie) Hash() (h common.Hash) { - hasher := newHasher(false) - defer returnHasherToPool(hasher) - - st.hashRec(hasher, nil) +func (t *StackTrie) Hash() (h common.Hash) { + st := t.root + t.hash(st, nil) if len(st.val) == 32 { copy(h[:], st.val) return h @@ -497,9 +377,9 @@ func (st *StackTrie) Hash() (h common.Hash) { // If the node's RLP isn't 32 bytes long, the node will not // be hashed, and instead contain the rlp-encoding of the // node. For the top level node, we need to force the hashing. - hasher.sha.Reset() - hasher.sha.Write(st.val) - hasher.sha.Read(h[:]) + t.h.sha.Reset() + t.h.sha.Write(st.val) + t.h.sha.Read(h[:]) return h } @@ -510,14 +390,12 @@ func (st *StackTrie) Hash() (h common.Hash) { // // The associated database is expected, otherwise the whole commit // functionality should be disabled. -func (st *StackTrie) Commit() (h common.Hash, err error) { - if st.writeFn == nil { +func (t *StackTrie) Commit() (h common.Hash, err error) { + if t.writeFn == nil { return common.Hash{}, ErrCommitDisabled } - hasher := newHasher(false) - defer returnHasherToPool(hasher) - - st.hashRec(hasher, nil) + st := t.root + t.hash(st, nil) if len(st.val) == 32 { copy(h[:], st.val) return h, nil @@ -525,10 +403,10 @@ func (st *StackTrie) Commit() (h common.Hash, err error) { // If the node's RLP isn't 32 bytes long, the node will not // be hashed (and committed), and instead contain the rlp-encoding of the // node. For the top level node, we need to force the hashing+commit. - hasher.sha.Reset() - hasher.sha.Write(st.val) - hasher.sha.Read(h[:]) + t.h.sha.Reset() + t.h.sha.Write(st.val) + t.h.sha.Read(h[:]) - st.writeFn(st.owner, nil, h, st.val) + t.writeFn(nil, h, st.val) return h, nil } diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index 6bd0b83e39..0e52781c62 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -198,12 +198,11 @@ func TestStackTrieInsertAndHash(t *testing.T) { {"000003", "XXXXXXXXXXXXXXXXXXXXXXXXXXXX", "962c0fffdeef7612a4f7bff1950d67e3e81c878e48b9ae45b3b374253b050bd8"}, }, } - st := NewStackTrie(nil) for i, test := range tests { // The StackTrie does not allow Insert(), Hash(), Insert(), ... // so we will create new trie for every sequence length of inserts. for l := 1; l <= len(test); l++ { - st.Reset() + st := NewStackTrie(nil) for j := 0; j < l; j++ { kv := &test[j] if err := st.Update(common.FromHex(kv.K), []byte(kv.V)); err != nil { @@ -377,48 +376,3 @@ func TestStacktrieNotModifyValues(t *testing.T) { } } } - -// TestStacktrieSerialization tests that the stacktrie works well if we -// serialize/unserialize it a lot -func TestStacktrieSerialization(t *testing.T) { - var ( - st = NewStackTrie(nil) - nt = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase(), nil)) - keyB = big.NewInt(1) - keyDelta = big.NewInt(1) - vals [][]byte - keys [][]byte - ) - getValue := func(i int) []byte { - if i%2 == 0 { // large - return crypto.Keccak256(big.NewInt(int64(i)).Bytes()) - } else { //small - return big.NewInt(int64(i)).Bytes() - } - } - for i := 0; i < 10; i++ { - vals = append(vals, getValue(i)) - keys = append(keys, common.BigToHash(keyB).Bytes()) - keyB = keyB.Add(keyB, keyDelta) - keyDelta.Add(keyDelta, common.Big1) - } - for i, k := range keys { - nt.Update(k, common.CopyBytes(vals[i])) - } - - for i, k := range keys { - blob, err := st.MarshalBinary() - if err != nil { - t.Fatal(err) - } - newSt, err := NewFromBinary(blob, nil) - if err != nil { - t.Fatal(err) - } - st = newSt - st.Update(k, common.CopyBytes(vals[i])) - } - if have, want := st.Hash(), nt.Hash(); have != want { - t.Fatalf("have %#x want %#x", have, want) - } -} diff --git a/trie/sync.go b/trie/sync.go index 9da0706075..6939aed76d 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -302,7 +302,7 @@ func (s *Sync) Missing(max int) ([]string, []common.Hash, []common.Hash) { } // ProcessCode injects the received data for requested item. Note it can -// happpen that the single response commits two pending requests(e.g. +// happen that the single response commits two pending requests(e.g. // there are two requests one for code and one for node but the hash // is same). In this case the second response for the same hash will // be treated as "non-requested" item or "already-processed" item but @@ -391,7 +391,7 @@ func (s *Sync) Pending() int { return len(s.nodeReqs) + len(s.codeReqs) } -// schedule inserts a new state retrieval request into the fetch queue. If there +// scheduleNodeRequest inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. func (s *Sync) scheduleNodeRequest(req *nodeRequest) { @@ -406,7 +406,7 @@ func (s *Sync) scheduleNodeRequest(req *nodeRequest) { s.queue.Push(string(req.path), prio) } -// schedule inserts a new state retrieval request into the fetch queue. If there +// scheduleCodeRequest inserts a new state retrieval request into the fetch queue. If there // is already a pending request for this node, the new request will be discarded // and only a parent reference added to the old one. func (s *Sync) scheduleCodeRequest(req *codeRequest) { @@ -556,7 +556,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { return requests, nil } -// commit finalizes a retrieval request and stores it into the membatch. If any +// commitNodeRequest finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. func (s *Sync) commitNodeRequest(req *nodeRequest) error { @@ -591,7 +591,7 @@ func (s *Sync) commitNodeRequest(req *nodeRequest) error { return nil } -// commit finalizes a retrieval request and stores it into the membatch. If any +// commitCodeRequest finalizes a retrieval request and stores it into the membatch. If any // of the referencing parent requests complete due to this commit, they are also // committed themselves. func (s *Sync) commitCodeRequest(req *codeRequest) error { diff --git a/trie/trie_test.go b/trie/trie_test.go index 8078770e7a..2dfe81ef81 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -912,8 +912,8 @@ func TestCommitSequenceStackTrie(t *testing.T) { trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} - stTrie := NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme()) + stTrie := NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) // Fill the trie with elements for i := 0; i < count; i++ { @@ -971,8 +971,8 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie := NewEmpty(db) // Another sponge is used for the stacktrie commits stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} - stTrie := NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { - rawdb.WriteTrieNode(stackTrieSponge, owner, path, hash, blob, db.Scheme()) + stTrie := NewStackTrie(func(path []byte, hash common.Hash, blob []byte) { + rawdb.WriteTrieNode(stackTrieSponge, common.Hash{}, path, hash, blob, db.Scheme()) }) // Add a single small-element to the trie(s) key := make([]byte, 5) diff --git a/light/nodeset.go b/trie/trienode/proof.go similarity index 73% rename from light/nodeset.go rename to trie/trienode/proof.go index 3662596785..012f0087dd 100644 --- a/light/nodeset.go +++ b/trie/trienode/proof.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package light +package trienode import ( "errors" @@ -26,9 +26,9 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// NodeSet stores a set of trie nodes. It implements trie.Database and can also +// ProofSet stores a set of trie nodes. It implements trie.Database and can also // act as a cache for another trie.Database. -type NodeSet struct { +type ProofSet struct { nodes map[string][]byte order []string @@ -36,15 +36,15 @@ type NodeSet struct { lock sync.RWMutex } -// NewNodeSet creates an empty node set -func NewNodeSet() *NodeSet { - return &NodeSet{ +// NewProofSet creates an empty node set +func NewProofSet() *ProofSet { + return &ProofSet{ nodes: make(map[string][]byte), } } // Put stores a new node in the set -func (db *NodeSet) Put(key []byte, value []byte) error { +func (db *ProofSet) Put(key []byte, value []byte) error { db.lock.Lock() defer db.lock.Unlock() @@ -61,7 +61,7 @@ func (db *NodeSet) Put(key []byte, value []byte) error { } // Delete removes a node from the set -func (db *NodeSet) Delete(key []byte) error { +func (db *ProofSet) Delete(key []byte) error { db.lock.Lock() defer db.lock.Unlock() @@ -70,7 +70,7 @@ func (db *NodeSet) Delete(key []byte) error { } // Get returns a stored node -func (db *NodeSet) Get(key []byte) ([]byte, error) { +func (db *ProofSet) Get(key []byte) ([]byte, error) { db.lock.RLock() defer db.lock.RUnlock() @@ -81,13 +81,13 @@ func (db *NodeSet) Get(key []byte) ([]byte, error) { } // Has returns true if the node set contains the given key -func (db *NodeSet) Has(key []byte) (bool, error) { +func (db *ProofSet) Has(key []byte) (bool, error) { _, err := db.Get(key) return err == nil, nil } // KeyCount returns the number of nodes in the set -func (db *NodeSet) KeyCount() int { +func (db *ProofSet) KeyCount() int { db.lock.RLock() defer db.lock.RUnlock() @@ -95,19 +95,19 @@ func (db *NodeSet) KeyCount() int { } // DataSize returns the aggregated data size of nodes in the set -func (db *NodeSet) DataSize() int { +func (db *ProofSet) DataSize() int { db.lock.RLock() defer db.lock.RUnlock() return db.dataSize } -// NodeList converts the node set to a NodeList -func (db *NodeSet) NodeList() NodeList { +// List converts the node set to a ProofList +func (db *ProofSet) List() ProofList { db.lock.RLock() defer db.lock.RUnlock() - var values NodeList + var values ProofList for _, key := range db.order { values = append(values, db.nodes[key]) } @@ -115,7 +115,7 @@ func (db *NodeSet) NodeList() NodeList { } // Store writes the contents of the set to the given database -func (db *NodeSet) Store(target ethdb.KeyValueWriter) { +func (db *ProofSet) Store(target ethdb.KeyValueWriter) { db.lock.RLock() defer db.lock.RUnlock() @@ -124,36 +124,36 @@ func (db *NodeSet) Store(target ethdb.KeyValueWriter) { } } -// NodeList stores an ordered list of trie nodes. It implements ethdb.KeyValueWriter. -type NodeList []rlp.RawValue +// ProofList stores an ordered list of trie nodes. It implements ethdb.KeyValueWriter. +type ProofList []rlp.RawValue // Store writes the contents of the list to the given database -func (n NodeList) Store(db ethdb.KeyValueWriter) { +func (n ProofList) Store(db ethdb.KeyValueWriter) { for _, node := range n { db.Put(crypto.Keccak256(node), node) } } -// NodeSet converts the node list to a NodeSet -func (n NodeList) NodeSet() *NodeSet { - db := NewNodeSet() +// Set converts the node list to a ProofSet +func (n ProofList) Set() *ProofSet { + db := NewProofSet() n.Store(db) return db } // Put stores a new node at the end of the list -func (n *NodeList) Put(key []byte, value []byte) error { +func (n *ProofList) Put(key []byte, value []byte) error { *n = append(*n, value) return nil } // Delete panics as there's no reason to remove a node from the list. -func (n *NodeList) Delete(key []byte) error { +func (n *ProofList) Delete(key []byte) error { panic("not supported") } // DataSize returns the aggregated data size of nodes in the list -func (n NodeList) DataSize() int { +func (n ProofList) DataSize() int { var size int for _, node := range n { size += len(node)