From f53981809ed494502d949a9e3fe0f3b4cd8c1ec5 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Thu, 14 Jul 2022 22:36:18 +0530 Subject: [PATCH 01/70] Change diverged flags back to Geth's convention (POS-602) (#451) * changed name -> identity * changed no-snapshot -> snapchot=true/false * changed jsonrpc.corsdomain -> http.corsdomain * changed jsonrpc.vhosts -> http.vhosts * changed http/ws.modules to http/ws.api * updated readme * updated config_test * make docs * handelling string array flag, overwrite insted of append * added 'Default' to SliceStringFlag * added separate flags for corsdomain and vhosts for http, ws, graphql * modified tests --- docs/cli/server.md | 20 +++-- internal/cli/flagset/flagset.go | 13 ++-- internal/cli/flagset/flagset_test.go | 11 ++- internal/cli/server/config.go | 64 ++++++++-------- internal/cli/server/config_test.go | 9 +-- internal/cli/server/flags.go | 105 ++++++++++++++++++--------- internal/cli/server/server.go | 4 +- 7 files changed, 138 insertions(+), 88 deletions(-) diff --git a/docs/cli/server.md b/docs/cli/server.md index 0803208e3a..ac43555275 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -6,7 +6,7 @@ The ```bor server``` command runs the Bor client. - ```chain```: Name of the chain to sync -- ```name```: Name/Identity of the node +- ```identity```: Name/Identity of the node - ```log-level```: Set log level for the server @@ -22,7 +22,7 @@ The ```bor server``` command runs the Bor client. - ```requiredblocks```: Comma separated block number-to-hash mappings to enforce (=) -- ```no-snapshot```: Disables the snapshot-database mode (default = false) +- ```snapshot```: Disables/Enables the snapshot-database mode (default = true) - ```bor.heimdall```: URL of Heimdall service @@ -88,9 +88,17 @@ The ```bor server``` command runs the Bor client. - ```ipcpath```: Filename for IPC socket/pipe within the datadir (explicit paths escape it) -- ```jsonrpc.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) +- ```http.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) -- ```jsonrpc.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. +- ```http.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. + +- ```ws.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) + +- ```ws.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. + +- ```graphql.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) + +- ```graphql.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. - ```http```: Enable the HTTP-RPC server @@ -100,7 +108,7 @@ The ```bor server``` command runs the Bor client. - ```http.rpcprefix```: HTTP path path prefix on which JSON-RPC is served. Use '/' to serve on all paths. -- ```http.modules```: API's offered over the HTTP-RPC interface +- ```http.api```: API's offered over the HTTP-RPC interface - ```ws```: Enable the WS-RPC server @@ -110,7 +118,7 @@ The ```bor server``` command runs the Bor client. - ```ws.rpcprefix```: HTTP path prefix on which JSON-RPC is served. Use '/' to serve on all paths. -- ```ws.modules```: API's offered over the WS-RPC interface +- ```ws.api```: API's offered over the WS-RPC interface - ```graphql```: Enable GraphQL on the HTTP-RPC server. Note that GraphQL can only be started if an HTTP server is started as well. diff --git a/internal/cli/flagset/flagset.go b/internal/cli/flagset/flagset.go index b57fff9996..0c19fa9758 100644 --- a/internal/cli/flagset/flagset.go +++ b/internal/cli/flagset/flagset.go @@ -202,10 +202,11 @@ func (f *Flagset) BigIntFlag(b *BigIntFlag) { } type SliceStringFlag struct { - Name string - Usage string - Value *[]string - Group string + Name string + Usage string + Value *[]string + Default []string + Group string } func (i *SliceStringFlag) String() string { @@ -217,8 +218,8 @@ func (i *SliceStringFlag) String() string { } func (i *SliceStringFlag) Set(value string) error { - *i.Value = append(*i.Value, strings.Split(value, ",")...) - + // overwritting insted of appending + *i.Value = strings.Split(value, ",") return nil } diff --git a/internal/cli/flagset/flagset_test.go b/internal/cli/flagset/flagset_test.go index 2f046c3248..118361320d 100644 --- a/internal/cli/flagset/flagset_test.go +++ b/internal/cli/flagset/flagset_test.go @@ -23,14 +23,17 @@ func TestFlagsetBool(t *testing.T) { func TestFlagsetSliceString(t *testing.T) { f := NewFlagSet("") - value := []string{} + value := []string{"a", "b", "c"} f.SliceStringFlag(&SliceStringFlag{ - Name: "flag", - Value: &value, + Name: "flag", + Value: &value, + Default: value, }) - assert.NoError(t, f.Parse([]string{"--flag", "a,b", "--flag", "c"})) + assert.NoError(t, f.Parse([]string{})) assert.Equal(t, value, []string{"a", "b", "c"}) + assert.NoError(t, f.Parse([]string{"--flag", "a,b"})) + assert.Equal(t, value, []string{"a", "b"}) } func TestFlagsetDuration(t *testing.T) { diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 347287e22f..fdf253a37e 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -40,8 +40,8 @@ type Config struct { // Chain is the chain to sync with Chain string `hcl:"chain,optional"` - // Name, or identity of the node - Name string `hcl:"name,optional"` + // Identity of the node + Identity string `hcl:"identity,optional"` // RequiredBlocks is a list of required (block number, hash) pairs to accept RequiredBlocks map[string]string `hcl:"requiredblocks,optional"` @@ -61,8 +61,8 @@ type Config struct { // GcMode selects the garbage collection mode for the trie GcMode string `hcl:"gc-mode,optional"` - // NoSnapshot disables the snapshot database mode - NoSnapshot bool `hcl:"no-snapshot,optional"` + // Snapshot disables/enables the snapshot database mode + Snapshot bool `hcl:"snapshot,optional"` // Ethstats is the address of the ethstats server to send telemetry Ethstats string `hcl:"ethstats,optional"` @@ -217,12 +217,6 @@ type JsonRPCConfig struct { // IPCPath is the path of the ipc endpoint IPCPath string `hcl:"ipc-path,optional"` - // VHost is the list of valid virtual hosts - VHost []string `hcl:"vhost,optional"` - - // Cors is the list of Cors endpoints - Cors []string `hcl:"cors,optional"` - // GasCap is the global gas cap for eth-call variants. GasCap uint64 `hcl:"gas-cap,optional"` @@ -232,10 +226,10 @@ type JsonRPCConfig struct { // Http has the json-rpc http related settings Http *APIConfig `hcl:"http,block"` - // Http has the json-rpc websocket related settings + // Ws has the json-rpc websocket related settings Ws *APIConfig `hcl:"ws,block"` - // Http has the json-rpc graphql related settings + // Graphql has the json-rpc graphql related settings Graphql *APIConfig `hcl:"graphql,block"` } @@ -258,7 +252,13 @@ type APIConfig struct { Host string `hcl:"host,optional"` // Modules is the list of enabled api modules - Modules []string `hcl:"modules,optional"` + API []string `hcl:"modules,optional"` + + // VHost is the list of valid virtual hosts + VHost []string `hcl:"vhost,optional"` + + // Cors is the list of Cors endpoints + Cors []string `hcl:"cors,optional"` } type GpoConfig struct { @@ -387,7 +387,7 @@ type DeveloperConfig struct { func DefaultConfig() *Config { return &Config{ Chain: "mainnet", - Name: Hostname(), + Identity: Hostname(), RequiredBlocks: map[string]string{}, LogLevel: "INFO", DataDir: defaultDataDir(), @@ -412,9 +412,9 @@ func DefaultConfig() *Config { URL: "http://localhost:1317", Without: false, }, - SyncMode: "full", - GcMode: "full", - NoSnapshot: false, + SyncMode: "full", + GcMode: "full", + Snapshot: true, TxPool: &TxPoolConfig{ Locals: []string{}, NoLocals: false, @@ -444,8 +444,6 @@ func DefaultConfig() *Config { JsonRPC: &JsonRPCConfig{ IPCDisable: false, IPCPath: "", - Cors: []string{"*"}, - VHost: []string{"*"}, GasCap: ethconfig.Defaults.RPCGasCap, TxFeeCap: ethconfig.Defaults.RPCTxFeeCap, Http: &APIConfig{ @@ -453,17 +451,23 @@ func DefaultConfig() *Config { Port: 8545, Prefix: "", Host: "localhost", - Modules: []string{"eth", "net", "web3", "txpool", "bor"}, + API: []string{"eth", "net", "web3", "txpool", "bor"}, + Cors: []string{"*"}, + VHost: []string{"*"}, }, Ws: &APIConfig{ Enabled: false, Port: 8546, Prefix: "", Host: "localhost", - Modules: []string{"web3", "net"}, + API: []string{"web3", "net"}, + Cors: []string{"*"}, + VHost: []string{"*"}, }, Graphql: &APIConfig{ Enabled: false, + Cors: []string{"*"}, + VHost: []string{"*"}, }, }, Ethstats: "", @@ -864,7 +868,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* } // snapshot disable check - if c.NoSnapshot { + if !c.Snapshot { if n.SyncMode == downloader.SnapSync { log.Info("Snap sync requested, enabling --snapshot") } else { @@ -908,15 +912,15 @@ func (c *Config) buildNode() (*node.Config, error) { ListenAddr: c.P2P.Bind + ":" + strconv.Itoa(int(c.P2P.Port)), DiscoveryV5: c.P2P.Discovery.V5Enabled, }, - HTTPModules: c.JsonRPC.Http.Modules, - HTTPCors: c.JsonRPC.Cors, - HTTPVirtualHosts: c.JsonRPC.VHost, + HTTPModules: c.JsonRPC.Http.API, + HTTPCors: c.JsonRPC.Http.Cors, + HTTPVirtualHosts: c.JsonRPC.Http.VHost, HTTPPathPrefix: c.JsonRPC.Http.Prefix, - WSModules: c.JsonRPC.Ws.Modules, - WSOrigins: c.JsonRPC.Cors, + WSModules: c.JsonRPC.Ws.API, + WSOrigins: c.JsonRPC.Ws.Cors, WSPathPrefix: c.JsonRPC.Ws.Prefix, - GraphQLCors: c.JsonRPC.Cors, - GraphQLVirtualHosts: c.JsonRPC.VHost, + GraphQLCors: c.JsonRPC.Graphql.Cors, + GraphQLVirtualHosts: c.JsonRPC.Graphql.VHost, } // dev mode @@ -999,7 +1003,7 @@ func (c *Config) buildNode() (*node.Config, error) { func (c *Config) Merge(cc ...*Config) error { for _, elem := range cc { - if err := mergo.Merge(c, elem, mergo.WithOverwriteWithEmptyValue, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(c, elem, mergo.WithOverwriteWithEmptyValue); err != nil { return fmt.Errorf("failed to merge configurations: %v", err) } } diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index 94c4496c03..c1c6c4ef4a 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -22,8 +22,8 @@ func TestConfigDefault(t *testing.T) { func TestConfigMerge(t *testing.T) { c0 := &Config{ - Chain: "0", - NoSnapshot: true, + Chain: "0", + Snapshot: true, RequiredBlocks: map[string]string{ "a": "b", }, @@ -54,8 +54,8 @@ func TestConfigMerge(t *testing.T) { } expected := &Config{ - Chain: "1", - NoSnapshot: false, + Chain: "1", + Snapshot: false, RequiredBlocks: map[string]string{ "a": "b", "b": "c", @@ -64,7 +64,6 @@ func TestConfigMerge(t *testing.T) { MaxPeers: 10, Discovery: &P2PDiscovery{ StaticNodes: []string{ - "a", "b", }, }, diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 21e9a38cb9..e85428b9ee 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -16,10 +16,10 @@ func (c *Command) Flags() *flagset.Flagset { Default: c.cliConfig.Chain, }) f.StringFlag(&flagset.StringFlag{ - Name: "name", + Name: "identity", Usage: "Name/Identity of the node", - Value: &c.cliConfig.Name, - Default: c.cliConfig.Name, + Value: &c.cliConfig.Identity, + Default: c.cliConfig.Identity, }) f.StringFlag(&flagset.StringFlag{ Name: "log-level", @@ -61,10 +61,10 @@ func (c *Command) Flags() *flagset.Flagset { Value: &c.cliConfig.RequiredBlocks, }) f.BoolFlag(&flagset.BoolFlag{ - Name: "no-snapshot", - Usage: `Disables the snapshot-database mode (default = false)`, - Value: &c.cliConfig.NoSnapshot, - Default: c.cliConfig.NoSnapshot, + Name: "snapshot", + Usage: `Disables/Enables the snapshot-database mode (default = true)`, + Value: &c.cliConfig.Snapshot, + Default: c.cliConfig.Snapshot, }) // heimdall @@ -83,10 +83,11 @@ func (c *Command) Flags() *flagset.Flagset { // txpool options f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "txpool.locals", - Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", - Value: &c.cliConfig.TxPool.Locals, - Group: "Transaction Pool", + Name: "txpool.locals", + Usage: "Comma separated accounts to treat as locals (no flush, priority inclusion)", + Value: &c.cliConfig.TxPool.Locals, + Default: c.cliConfig.TxPool.Locals, + Group: "Transaction Pool", }) f.BoolFlag(&flagset.BoolFlag{ Name: "txpool.nolocals", @@ -329,16 +330,46 @@ func (c *Command) Flags() *flagset.Flagset { Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "jsonrpc.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: &c.cliConfig.JsonRPC.Cors, - Group: "JsonRPC", + Name: "http.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: &c.cliConfig.JsonRPC.Http.Cors, + Default: c.cliConfig.JsonRPC.Http.Cors, + Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "jsonrpc.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: &c.cliConfig.JsonRPC.VHost, - Group: "JsonRPC", + Name: "http.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: &c.cliConfig.JsonRPC.Http.VHost, + Default: c.cliConfig.JsonRPC.Http.VHost, + Group: "JsonRPC", + }) + f.SliceStringFlag(&flagset.SliceStringFlag{ + Name: "ws.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: &c.cliConfig.JsonRPC.Ws.Cors, + Default: c.cliConfig.JsonRPC.Ws.Cors, + Group: "JsonRPC", + }) + f.SliceStringFlag(&flagset.SliceStringFlag{ + Name: "ws.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: &c.cliConfig.JsonRPC.Ws.VHost, + Default: c.cliConfig.JsonRPC.Ws.VHost, + Group: "JsonRPC", + }) + f.SliceStringFlag(&flagset.SliceStringFlag{ + Name: "graphql.corsdomain", + Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", + Value: &c.cliConfig.JsonRPC.Graphql.Cors, + Default: c.cliConfig.JsonRPC.Graphql.Cors, + Group: "JsonRPC", + }) + f.SliceStringFlag(&flagset.SliceStringFlag{ + Name: "graphql.vhosts", + Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", + Value: &c.cliConfig.JsonRPC.Graphql.VHost, + Default: c.cliConfig.JsonRPC.Graphql.VHost, + Group: "JsonRPC", }) // http options @@ -371,10 +402,11 @@ func (c *Command) Flags() *flagset.Flagset { Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "http.modules", - Usage: "API's offered over the HTTP-RPC interface", - Value: &c.cliConfig.JsonRPC.Http.Modules, - Group: "JsonRPC", + Name: "http.api", + Usage: "API's offered over the HTTP-RPC interface", + Value: &c.cliConfig.JsonRPC.Http.API, + Default: c.cliConfig.JsonRPC.Http.API, + Group: "JsonRPC", }) // ws options @@ -407,10 +439,11 @@ func (c *Command) Flags() *flagset.Flagset { Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "ws.modules", - Usage: "API's offered over the WS-RPC interface", - Value: &c.cliConfig.JsonRPC.Ws.Modules, - Group: "JsonRPC", + Name: "ws.api", + Usage: "API's offered over the WS-RPC interface", + Value: &c.cliConfig.JsonRPC.Ws.API, + Default: c.cliConfig.JsonRPC.Ws.API, + Group: "JsonRPC", }) // graphql options @@ -438,10 +471,11 @@ func (c *Command) Flags() *flagset.Flagset { Group: "P2P", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "bootnodes", - Usage: "Comma separated enode URLs for P2P discovery bootstrap", - Value: &c.cliConfig.P2P.Discovery.Bootnodes, - Group: "P2P", + Name: "bootnodes", + Usage: "Comma separated enode URLs for P2P discovery bootstrap", + Value: &c.cliConfig.P2P.Discovery.Bootnodes, + Default: c.cliConfig.P2P.Discovery.Bootnodes, + Group: "P2P", }) f.Uint64Flag(&flagset.Uint64Flag{ Name: "maxpeers", @@ -581,10 +615,11 @@ func (c *Command) Flags() *flagset.Flagset { // account f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "unlock", - Usage: "Comma separated list of accounts to unlock", - Value: &c.cliConfig.Accounts.Unlock, - Group: "Account Management", + Name: "unlock", + Usage: "Comma separated list of accounts to unlock", + Value: &c.cliConfig.Accounts.Unlock, + Default: c.cliConfig.Accounts.Unlock, + Group: "Account Management", }) f.StringFlag(&flagset.StringFlag{ Name: "password", diff --git a/internal/cli/server/server.go b/internal/cli/server/server.go index 7d5ec2c8f3..cd706c1a9d 100644 --- a/internal/cli/server/server.go +++ b/internal/cli/server/server.go @@ -182,7 +182,7 @@ func NewServer(config *Config) (*Server, error) { // graphql is started from another place if config.JsonRPC.Graphql.Enabled { - if err := graphql.New(stack, srv.backend.APIBackend, config.JsonRPC.Cors, config.JsonRPC.VHost); err != nil { + if err := graphql.New(stack, srv.backend.APIBackend, config.JsonRPC.Graphql.Cors, config.JsonRPC.Graphql.VHost); err != nil { return nil, fmt.Errorf("failed to register the GraphQL service: %v", err) } } @@ -201,7 +201,7 @@ func NewServer(config *Config) (*Server, error) { } } - if err := srv.setupMetrics(config.Telemetry, config.Name); err != nil { + if err := srv.setupMetrics(config.Telemetry, config.Identity); err != nil { return nil, err } From e62beee1b2dda5c4ab9277ea351c9e4b73125c2f Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Wed, 27 Jul 2022 12:28:26 +0530 Subject: [PATCH 02/70] internal/cli: use config file for flags in new-cli, builder/files: update defaults (#457) * updated simple.json and simple.hcl * added annotations for developer and grpc block * added toml tags and simple.toml file * added support for toml config files * updated simple files toml, hcl, json * added config.toml in builder/files and updated bor.service * add dumpconfig command in cli for exporting configs * update docs * updated .goreleaser.yml (POS-651) * changed --config to -config * added test config for tests and fixed lint errors * made fields of type big.int and time.Duration private, removed merge from dumpconfig, setting up default values to the Raw fields in dumpconfig, and fixed one lint error * fixed lint errors (strange) * lint fix * private no-more, using '-' in name tags to ignore * updated name tags, made c.configFile as a stringFlag (only one config file supported) and updated the merge logic in command.go -> Run() * fix: set method for big.Int flags, added a TODO * handeled bigInt and timeDuration type, bug fix in config_legacy, lint fix * updated flags, consistent across flags.go and config.go * fixed config test and updated test hcl, json config files * updated config legacy test * added test to check values of cmd flags, restructured Run in command.go, linter fix * fix linters * lint again * changed 2 flags and assert -> require * changed the 2 flags back * updated correct congig.toml path and made mainnet default * updated config.toml with new flags * added sample config (toml) file * removed sample-config.toml and added it in docs/config.md Co-authored-by: Manav Darji --- .goreleaser.yml | 3 + builder/files/bor.service | 29 +- builder/files/config.toml | 44 ++++ docs/cli/README.md | 2 + docs/cli/dumpconfig.md | 3 + docs/config.md | 235 +++++++++-------- go.mod | 1 + go.sum | 6 + internal/cli/command.go | 5 + internal/cli/dumpconfig.go | 82 ++++++ internal/cli/flagset/flagset.go | 18 +- internal/cli/server/command.go | 57 +++- internal/cli/server/command_test.go | 50 ++++ internal/cli/server/config.go | 249 +++++++++--------- internal/cli/server/config_legacy.go | 40 +-- internal/cli/server/config_legacy_test.go | 45 +++- internal/cli/server/config_test.go | 6 +- internal/cli/server/flags.go | 2 +- internal/cli/server/proto/server.pb.go | 5 +- internal/cli/server/proto/server_grpc.pb.go | 1 + internal/cli/server/service_test.go | 3 +- internal/cli/server/testdata/simple.json | 15 -- .../server/testdata/{simple.hcl => test.hcl} | 8 +- internal/cli/server/testdata/test.json | 15 ++ internal/cli/server/testdata/test.toml | 25 ++ 25 files changed, 612 insertions(+), 337 deletions(-) create mode 100644 builder/files/config.toml create mode 100644 docs/cli/dumpconfig.md create mode 100644 internal/cli/dumpconfig.go create mode 100644 internal/cli/server/command_test.go delete mode 100644 internal/cli/server/testdata/simple.json rename internal/cli/server/testdata/{simple.hcl => test.hcl} (57%) create mode 100644 internal/cli/server/testdata/test.json create mode 100644 internal/cli/server/testdata/test.toml diff --git a/.goreleaser.yml b/.goreleaser.yml index d09cc43206..7fbc61ca4a 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -90,6 +90,9 @@ nfpms: - src: builder/files/genesis-testnet-v4.json dst: /etc/bor/genesis-testnet-v4.json type: config + - src: builder/files/config.toml + dst: /var/lib/bor/config.toml + type: config overrides: rpm: diff --git a/builder/files/bor.service b/builder/files/bor.service index da0338368c..4b628cbf6e 100644 --- a/builder/files/bor.service +++ b/builder/files/bor.service @@ -6,34 +6,7 @@ [Service] Restart=on-failure RestartSec=5s - ExecStart=/usr/local/bin/bor server \ - --chain=mumbai \ - # --chain=mainnet \ - --datadir /var/lib/bor/data \ - --metrics \ - --metrics.prometheus-addr="127.0.0.1:7071" \ - --syncmode 'full' \ - --miner.gasprice '30000000000' \ - --miner.gaslimit '20000000' \ - --miner.gastarget '20000000' \ - --txpool.nolocals \ - --txpool.accountslots 16 \ - --txpool.globalslots 32768 \ - --txpool.accountqueue 16 \ - --txpool.globalqueue 32768 \ - --txpool.pricelimit '30000000000' \ - --txpool.lifetime '1h30m0s' \ - --bootnodes "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303,enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303" - # Validator params - # Uncomment and configure the following lines in case you run a validator. Don't forget to add backslash (\) - # to previous command line. - # --keystore /var/lib/bor/keystore \ - # --unlock [VALIDATOR ADDRESS] \ - # --password /var/lib/bor/password.txt \ - # --allow-insecure-unlock \ - # --nodiscover --maxpeers 1 \ - # --miner.etherbase [VALIDATOR ADDRESS] \ - # --mine + ExecStart=/usr/local/bin/bor server -config="/var/lib/bor/config.toml" Type=simple User=ubuntu KillSignal=SIGINT diff --git a/builder/files/config.toml b/builder/files/config.toml new file mode 100644 index 0000000000..c55a143ed7 --- /dev/null +++ b/builder/files/config.toml @@ -0,0 +1,44 @@ +# chain = "mumbai" +chain = "mainnet" +datadir = "/var/lib/bor/data" +syncmode = "full" + +[telemetry] +metrics = true +prometheus-addr = "127.0.0.1:7071" + +[miner] +# *** Validator params +# *** Uncomment and configure the following lines in case you run a validator. +# mine = true +# etherbase = "VALIDATOR ADDRESS" +gasprice = "30000000000" +gasceil = 20000000 + + +[txpool] +accountqueue = 16 +accountslots = 16 +globalqueue = 32768 +globalslots = 32768 +lifetime = "1h30m0s" +nolocals = true +pricelimit = 30000000000 + +[p2p] +# *** Validator params +# *** Uncomment and configure the following lines in case you run a validator. +# nodiscover = true +# maxpeers = 1 + [p2p.discovery] + bootnodes = ["enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303", "enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303"] + +# *** Validator params +# *** Uncomment and configure the following lines in case you run a validator. + +# keystore = "/var/lib/bor/keystore" + +# [accounts] +# allow-insecure-unlock = true +# password = "/var/lib/bor/password.txt" +# unlock = ["VALIDATOR ADDRESS"] \ No newline at end of file diff --git a/docs/cli/README.md b/docs/cli/README.md index d92a7b5a05..3bc3daddc5 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -26,6 +26,8 @@ - [```debug pprof```](./debug_pprof.md) +- [```dumpconfig```](./dumpconfig.md) + - [```fingerprint```](./fingerprint.md) - [```peers```](./peers.md) diff --git a/docs/cli/dumpconfig.md b/docs/cli/dumpconfig.md new file mode 100644 index 0000000000..0383c47310 --- /dev/null +++ b/docs/cli/dumpconfig.md @@ -0,0 +1,3 @@ +# Dumpconfig + +The ```bor dumpconfig ``` command will export the user provided flags into a configuration file \ No newline at end of file diff --git a/docs/config.md b/docs/config.md index 4f4dec157b..196a0bf388 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1,133 +1,144 @@ # Config -Toml files format used in geth are being deprecated. - -Bor uses uses JSON and [HCL](https://github.com/hashicorp/hcl) formats to create configuration files. This is the format in HCL alongside the default values: - +- The `bor dumpconfig` command prints the default configurations, in the TOML format, on the terminal. + - One can `pipe (>)` this to a file (say `config.toml`) and use it to start bor. + - Command to provide a config file: `bor server -config config.toml` +- Bor uses TOML, HCL, and JSON format config files. +- This is the format of the config file in TOML: + - **NOTE: The values of these following flags are just for reference** + - `config.toml` file: ``` chain = "mainnet" -log-level = "info" -data-dir = "" -sync-mode = "fast" -gc-mode = "full" +identity = "myIdentity" +log-level = "INFO" +datadir = "/var/lib/bor/data" +keystore = "path/to/keystore" +syncmode = "full" +gcmode = "full" snapshot = true ethstats = "" -whitelist = {} -p2p { - max-peers = 30 - max-pend-peers = 50 - bind = "0.0.0.0" - port = 30303 - no-discover = false - nat = "any" - discovery { - v5-enabled = false - bootnodes = [] - bootnodesv4 = [] - bootnodesv5 = [] - staticNodes = [] - trustedNodes = [] - dns = [] - } -} +[p2p] +maxpeers = 30 +maxpendpeers = 50 +bind = "0.0.0.0" +port = 30303 +nodiscover = false +nat = "any" -heimdall { - url = "http://localhost:1317" - without = false -} +[p2p.discovery] +v5disc = false +bootnodes = ["enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", "enode://22a8232c3abc76a16ae9d6c3b164f98775fe226f0917b0ca871128a74a8e9630b458460865bab457221f1d448dd9791d24c4e5d88786180ac185df813a68d4de@3.209.45.79:30303"] +bootnodesv4 = [] +bootnodesv5 = ["enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", "enr:-KG4QDyytgmE4f7AnvW-ZaUOIi9i79qX4JwjRAiXBZCU65wOfBu-3Nb5I7b_Rmg3KCOcZM_C3y5pg7EBU5XGrcLTduQEhGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQ2_DUbiXNlY3AyNTZrMaEDKnz_-ps3UUOfHWVYaskI5kWYO_vtYMGYCQRAR3gHDouDdGNwgiMog3VkcIIjKA"] +static-nodes = ["enode://8499da03c47d637b20eee24eec3c356c9a2e6148d6fe25ca195c7949ab8ec2c03e3556126b0d7ed644675e78c4318b08691b7b57de10e5f0d40d05b09238fa0a@52.187.207.27:30303"] +trusted-nodes = ["enode://2b252ab6a1d0f971d9722cb839a42cb81db019ba44c08754628ab4a823487071b5695317c8ccd085219c3a03af063495b2f1da8d18218da2d6a82981b45e6ffc@65.108.70.101:30303"] +dns = [] -txpool { - locals = [] - no-locals = false - journal = "" - rejournal = "1h" - price-limit = 1 - price-bump = 10 - account-slots = 16 - global-slots = 4096 - account-queue = 64 - global-queue = 1024 - lifetime = "3h" -} +[heimdall] +url = "http://localhost:1317" +"bor.without" = false -sealer { - enabled = false - etherbase = "" - gas-ceil = 8000000 - extra-data = "" -} +[txpool] +locals = ["$ADDRESS1", "$ADDRESS2"] +nolocals = false +journal = "" +rejournal = "1h0m0s" +pricelimit = 30000000000 +pricebump = 10 +accountslots = 16 +globalslots = 32768 +accountqueue = 16 +globalqueue = 32768 +lifetime = "3h0m0s" -gpo { - blocks = 20 - percentile = 60 -} +[miner] +mine = false +etherbase = "" +extradata = "" +gaslimit = 20000000 +gasprice = "30000000000" -jsonrpc { - ipc-disable = false - ipc-path = "" - modules = ["web3", "net"] - cors = ["*"] - vhost = ["*"] - - http { - enabled = false - port = 8545 - prefix = "" - host = "localhost" - } +[jsonrpc] +ipcdisable = false +ipcpath = "/var/lib/bor/bor.ipc" +gascap = 50000000 +txfeecap = 5e+00 - ws { - enabled = false - port = 8546 - prefix = "" - host = "localhost" - } +[jsonrpc.http] +enabled = false +port = 8545 +prefix = "" +host = "localhost" +api = ["eth", "net", "web3", "txpool", "bor"] +vhosts = ["*"] +corsdomain = ["*"] - graphqh { - enabled = false - } -} +[jsonrpc.ws] +enabled = false +port = 8546 +prefix = "" +host = "localhost" +api = ["web3", "net"] +vhosts = ["*"] +corsdomain = ["*"] -telemetry { - enabled = false - expensive = false +[jsonrpc.graphql] +enabled = false +port = 0 +prefix = "" +host = "" +api = [] +vhosts = ["*"] +corsdomain = ["*"] - influxdb { - v1-enabled = false - endpoint = "" - database = "" - username = "" - password = "" - v2-enabled = false - token = "" - bucket = "" - organization = "" - } -} +[gpo] +blocks = 20 +percentile = 60 +maxprice = "5000000000000" +ignoreprice = "2" -cache { - cache = 1024 - perc-database = 50 - perc-trie = 15 - perc-gc = 25 - perc-snapshot = 10 - journal = "triecache" - rejournal = "60m" - no-prefetch = false - preimages = false - tx-lookup-limit = 2350000 -} +[telemetry] +metrics = false +expensive = false +prometheus-addr = "" +opencollector-endpoint = "" -accounts { - unlock = [] - password-file = "" - allow-insecure-unlock = false - use-lightweight-kdf = false -} +[telemetry.influx] +influxdb = false +endpoint = "" +database = "" +username = "" +password = "" +influxdbv2 = false +token = "" +bucket = "" +organization = "" -grpc { - addr = ":3131" -} +[cache] +cache = 1024 +gc = 25 +snapshot = 10 +database = 50 +trie = 15 +journal = "triecache" +rejournal = "1h0m0s" +noprefetch = false +preimages = false +txlookuplimit = 2350000 + +[accounts] +unlock = ["$ADDRESS1", "$ADDRESS2"] +password = "path/to/password.txt" +allow-insecure-unlock = false +lightkdf = false +disable-bor-wallet = false + +[grpc] +addr = ":3131" + +[developer] +dev = false +period = 0 ``` diff --git a/go.mod b/go.mod index 1b452a67a1..3f90695d47 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/BurntSushi/toml v1.1.0 // indirect github.com/Masterminds/goutils v1.1.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect diff --git a/go.sum b/go.sum index 374ab3066e..fc58b621e6 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= @@ -456,6 +458,10 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= diff --git a/internal/cli/command.go b/internal/cli/command.go index 1ff95b410f..06127a9823 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -82,6 +82,11 @@ func Commands() map[string]MarkDownCommandFactory { UI: ui, }, nil }, + "dumpconfig": func() (MarkDownCommand, error) { + return &DumpconfigCommand{ + Meta2: meta2, + }, nil + }, "debug": func() (MarkDownCommand, error) { return &DebugCommand{ UI: ui, diff --git a/internal/cli/dumpconfig.go b/internal/cli/dumpconfig.go new file mode 100644 index 0000000000..3e4688fc24 --- /dev/null +++ b/internal/cli/dumpconfig.go @@ -0,0 +1,82 @@ +package cli + +import ( + "reflect" + "strings" + + "github.com/naoina/toml" + + "github.com/ethereum/go-ethereum/internal/cli/server" +) + +// These settings ensure that TOML keys use the same names as Go struct fields. +var tomlSettings = toml.Config{ + NormFieldName: func(rt reflect.Type, key string) string { + return key + }, + FieldToKey: func(rt reflect.Type, field string) string { + return field + }, +} + +// DumpconfigCommand is for exporting user provided flags into a config file +type DumpconfigCommand struct { + *Meta2 +} + +// MarkDown implements cli.MarkDown interface +func (p *DumpconfigCommand) MarkDown() string { + items := []string{ + "# Dumpconfig", + "The ```bor dumpconfig ``` command will export the user provided flags into a configuration file", + } + + return strings.Join(items, "\n\n") +} + +// Help implements the cli.Command interface +func (c *DumpconfigCommand) Help() string { + return `Usage: bor dumpconfig + + This command will will export the user provided flags into a configuration file` +} + +// Synopsis implements the cli.Command interface +func (c *DumpconfigCommand) Synopsis() string { + return "Export configuration file" +} + +// TODO: add flags for file location and format (toml, json, hcl) of the configuration file. + +// Run implements the cli.Command interface +func (c *DumpconfigCommand) Run(args []string) int { + // Initialize an empty command instance to get flags + command := server.Command{} + flags := command.Flags() + + if err := flags.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + userConfig := command.GetConfig() + + // convert the big.Int and time.Duration fields to their corresponding Raw fields + userConfig.TxPool.RejournalRaw = userConfig.TxPool.Rejournal.String() + userConfig.TxPool.LifeTimeRaw = userConfig.TxPool.LifeTime.String() + userConfig.Sealer.GasPriceRaw = userConfig.Sealer.GasPrice.String() + userConfig.Gpo.MaxPriceRaw = userConfig.Gpo.MaxPrice.String() + userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String() + userConfig.Cache.RejournalRaw = userConfig.Cache.Rejournal.String() + + // Currently, the configurations (userConfig) is exported into `toml` file format. + out, err := tomlSettings.Marshal(&userConfig) + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + + c.UI.Output(string(out)) + + return 0 +} diff --git a/internal/cli/flagset/flagset.go b/internal/cli/flagset/flagset.go index 0c19fa9758..d9e204f0e4 100644 --- a/internal/cli/flagset/flagset.go +++ b/internal/cli/flagset/flagset.go @@ -180,14 +180,15 @@ func (b *BigIntFlag) Set(value string) error { var ok bool if strings.HasPrefix(value, "0x") { num, ok = num.SetString(value[2:], 16) + *b.Value = *num } else { num, ok = num.SetString(value, 10) + *b.Value = *num } if !ok { return fmt.Errorf("failed to set big int") } - b.Value = num return nil } @@ -209,6 +210,19 @@ type SliceStringFlag struct { Group string } +// SplitAndTrim splits input separated by a comma +// and trims excessive white space from the substrings. +func SplitAndTrim(input string) (ret []string) { + l := strings.Split(input, ",") + for _, r := range l { + if r = strings.TrimSpace(r); r != "" { + ret = append(ret, r) + } + } + + return ret +} + func (i *SliceStringFlag) String() string { if i.Value == nil { return "" @@ -219,7 +233,7 @@ func (i *SliceStringFlag) String() string { func (i *SliceStringFlag) Set(value string) error { // overwritting insted of appending - *i.Value = strings.Split(value, ",") + *i.Value = SplitAndTrim(value) return nil } diff --git a/internal/cli/server/command.go b/internal/cli/server/command.go index a0b8a6d87c..2995f10f69 100644 --- a/internal/cli/server/command.go +++ b/internal/cli/server/command.go @@ -7,8 +7,9 @@ import ( "strings" "syscall" - "github.com/ethereum/go-ethereum/log" "github.com/mitchellh/cli" + + "github.com/ethereum/go-ethereum/log" ) // Command is the command to start the sever @@ -21,7 +22,7 @@ type Command struct { // final configuration config *Config - configFile []string + configFile string srv *Server } @@ -50,34 +51,57 @@ func (c *Command) Synopsis() string { return "Run the Bor server" } -// Run implements the cli.Command interface -func (c *Command) Run(args []string) int { +func (c *Command) extractFlags(args []string) error { + config := *DefaultConfig() + flags := c.Flags() if err := flags.Parse(args); err != nil { c.UI.Error(err.Error()) - return 1 + c.config = &config + + return err } - // read config file - config := DefaultConfig() - for _, configFile := range c.configFile { - cfg, err := readConfigFile(configFile) + // TODO: Check if this can be removed or not + // read cli flags + if err := config.Merge(c.cliConfig); err != nil { + c.UI.Error(err.Error()) + c.config = &config + + return err + } + // read if config file is provided, this will overwrite the cli flags, if provided + if c.configFile != "" { + log.Warn("Config File provided, this will overwrite the cli flags.", "configFile:", c.configFile) + cfg, err := readConfigFile(c.configFile) if err != nil { c.UI.Error(err.Error()) - return 1 + c.config = &config + + return err } if err := config.Merge(cfg); err != nil { c.UI.Error(err.Error()) - return 1 + c.config = &config + + return err } } - if err := config.Merge(c.cliConfig); err != nil { + + c.config = &config + + return nil +} + +// Run implements the cli.Command interface +func (c *Command) Run(args []string) int { + err := c.extractFlags(args) + if err != nil { c.UI.Error(err.Error()) return 1 } - c.config = config - srv, err := NewServer(config) + srv, err := NewServer(c.config) if err != nil { c.UI.Error(err.Error()) return 1 @@ -112,3 +136,8 @@ func (c *Command) handleSignals() int { } return 1 } + +// GetConfig returns the user specified config +func (c *Command) GetConfig() *Config { + return c.cliConfig +} diff --git a/internal/cli/server/command_test.go b/internal/cli/server/command_test.go new file mode 100644 index 0000000000..9006559d0f --- /dev/null +++ b/internal/cli/server/command_test.go @@ -0,0 +1,50 @@ +package server + +import ( + "math/big" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestFlags(t *testing.T) { + t.Parallel() + + var c Command + + args := []string{ + "--txpool.rejournal", "30m0s", + "--txpool.lifetime", "30m0s", + "--miner.gasprice", "20000000000", + "--gpo.maxprice", "70000000000", + "--gpo.ignoreprice", "1", + "--cache.trie.rejournal", "40m0s", + "--dev", + "--dev.period", "2", + "--datadir", "./data", + "--maxpeers", "30", + "--requiredblocks", "a=b", + "--http.api", "eth,web3,bor", + } + err := c.extractFlags(args) + + require.NoError(t, err) + + txRe, _ := time.ParseDuration("30m0s") + txLt, _ := time.ParseDuration("30m0s") + caRe, _ := time.ParseDuration("40m0s") + + require.Equal(t, c.config.DataDir, "./data") + require.Equal(t, c.config.Developer.Enabled, true) + require.Equal(t, c.config.Developer.Period, uint64(2)) + require.Equal(t, c.config.TxPool.Rejournal, txRe) + require.Equal(t, c.config.TxPool.LifeTime, txLt) + require.Equal(t, c.config.Sealer.GasPrice, big.NewInt(20000000000)) + require.Equal(t, c.config.Gpo.MaxPrice, big.NewInt(70000000000)) + require.Equal(t, c.config.Gpo.IgnorePrice, big.NewInt(1)) + require.Equal(t, c.config.Cache.Rejournal, caRe) + require.Equal(t, c.config.P2P.MaxPeers, uint64(30)) + require.Equal(t, c.config.RequiredBlocks, map[string]string{"a": "b"}) + require.Equal(t, c.config.JsonRPC.Http.API, []string{"eth", "web3", "bor"}) +} diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index fdf253a37e..819ab1aca5 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -14,6 +14,11 @@ import ( godebug "runtime/debug" + "github.com/hashicorp/hcl/v2/hclsimple" + "github.com/imdario/mergo" + "github.com/mitchellh/go-homedir" + gopsutil "github.com/shirou/gopsutil/mem" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/common" @@ -28,360 +33,356 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/params" - "github.com/hashicorp/hcl/v2/hclsimple" - "github.com/imdario/mergo" - "github.com/mitchellh/go-homedir" - gopsutil "github.com/shirou/gopsutil/mem" ) type Config struct { chain *chains.Chain // Chain is the chain to sync with - Chain string `hcl:"chain,optional"` + Chain string `hcl:"chain,optional" toml:"chain,optional"` // Identity of the node - Identity string `hcl:"identity,optional"` + Identity string `hcl:"identity,optional" toml:"identity,optional"` // RequiredBlocks is a list of required (block number, hash) pairs to accept - RequiredBlocks map[string]string `hcl:"requiredblocks,optional"` + RequiredBlocks map[string]string `hcl:"requiredblocks,optional" toml:"requiredblocks,optional"` // LogLevel is the level of the logs to put out - LogLevel string `hcl:"log-level,optional"` + LogLevel string `hcl:"log-level,optional" toml:"log-level,optional"` // DataDir is the directory to store the state in - DataDir string `hcl:"data-dir,optional"` + DataDir string `hcl:"datadir,optional" toml:"datadir,optional"` // KeyStoreDir is the directory to store keystores - KeyStoreDir string `hcl:"keystore-dir,optional"` + KeyStoreDir string `hcl:"keystore,optional" toml:"keystore,optional"` // SyncMode selects the sync protocol - SyncMode string `hcl:"sync-mode,optional"` + SyncMode string `hcl:"syncmode,optional" toml:"syncmode,optional"` // GcMode selects the garbage collection mode for the trie - GcMode string `hcl:"gc-mode,optional"` + GcMode string `hcl:"gcmode,optional" toml:"gcmode,optional"` // Snapshot disables/enables the snapshot database mode - Snapshot bool `hcl:"snapshot,optional"` + Snapshot bool `hcl:"snapshot,optional" toml:"snapshot,optional"` // Ethstats is the address of the ethstats server to send telemetry - Ethstats string `hcl:"ethstats,optional"` + Ethstats string `hcl:"ethstats,optional" toml:"ethstats,optional"` // P2P has the p2p network related settings - P2P *P2PConfig `hcl:"p2p,block"` + P2P *P2PConfig `hcl:"p2p,block" toml:"p2p,block"` // Heimdall has the heimdall connection related settings - Heimdall *HeimdallConfig `hcl:"heimdall,block"` + Heimdall *HeimdallConfig `hcl:"heimdall,block" toml:"heimdall,block"` // TxPool has the transaction pool related settings - TxPool *TxPoolConfig `hcl:"txpool,block"` + TxPool *TxPoolConfig `hcl:"txpool,block" toml:"txpool,block"` // Sealer has the validator related settings - Sealer *SealerConfig `hcl:"sealer,block"` + Sealer *SealerConfig `hcl:"miner,block" toml:"miner,block"` // JsonRPC has the json-rpc related settings - JsonRPC *JsonRPCConfig `hcl:"jsonrpc,block"` + JsonRPC *JsonRPCConfig `hcl:"jsonrpc,block" toml:"jsonrpc,block"` // Gpo has the gas price oracle related settings - Gpo *GpoConfig `hcl:"gpo,block"` + Gpo *GpoConfig `hcl:"gpo,block" toml:"gpo,block"` // Telemetry has the telemetry related settings - Telemetry *TelemetryConfig `hcl:"telemetry,block"` + Telemetry *TelemetryConfig `hcl:"telemetry,block" toml:"telemetry,block"` // Cache has the cache related settings - Cache *CacheConfig `hcl:"cache,block"` + Cache *CacheConfig `hcl:"cache,block" toml:"cache,block"` // Account has the validator account related settings - Accounts *AccountsConfig `hcl:"accounts,block"` + Accounts *AccountsConfig `hcl:"accounts,block" toml:"accounts,block"` // GRPC has the grpc server related settings - GRPC *GRPCConfig + GRPC *GRPCConfig `hcl:"grpc,block" toml:"grpc,block"` // Developer has the developer mode related settings - Developer *DeveloperConfig + Developer *DeveloperConfig `hcl:"developer,block" toml:"developer,block"` } type P2PConfig struct { // MaxPeers sets the maximum number of connected peers - MaxPeers uint64 `hcl:"max-peers,optional"` + MaxPeers uint64 `hcl:"maxpeers,optional" toml:"maxpeers,optional"` // MaxPendPeers sets the maximum number of pending connected peers - MaxPendPeers uint64 `hcl:"max-pend-peers,optional"` + MaxPendPeers uint64 `hcl:"maxpendpeers,optional" toml:"maxpendpeers,optional"` // Bind is the bind address - Bind string `hcl:"bind,optional"` + Bind string `hcl:"bind,optional" toml:"bind,optional"` // Port is the port number - Port uint64 `hcl:"port,optional"` + Port uint64 `hcl:"port,optional" toml:"port,optional"` // NoDiscover is used to disable discovery - NoDiscover bool `hcl:"no-discover,optional"` + NoDiscover bool `hcl:"nodiscover,optional" toml:"nodiscover,optional"` // NAT it used to set NAT options - NAT string `hcl:"nat,optional"` + NAT string `hcl:"nat,optional" toml:"nat,optional"` // Discovery has the p2p discovery related settings - Discovery *P2PDiscovery `hcl:"discovery,block"` + Discovery *P2PDiscovery `hcl:"discovery,block" toml:"discovery,block"` } type P2PDiscovery struct { // V5Enabled is used to enable disc v5 discovery mode - V5Enabled bool `hcl:"v5-enabled,optional"` + V5Enabled bool `hcl:"v5disc,optional" toml:"v5disc,optional"` // Bootnodes is the list of initial bootnodes - Bootnodes []string `hcl:"bootnodes,optional"` + Bootnodes []string `hcl:"bootnodes,optional" toml:"bootnodes,optional"` // BootnodesV4 is the list of initial v4 bootnodes - BootnodesV4 []string `hcl:"bootnodesv4,optional"` + BootnodesV4 []string `hcl:"bootnodesv4,optional" toml:"bootnodesv4,optional"` // BootnodesV5 is the list of initial v5 bootnodes - BootnodesV5 []string `hcl:"bootnodesv5,optional"` + BootnodesV5 []string `hcl:"bootnodesv5,optional" toml:"bootnodesv5,optional"` // StaticNodes is the list of static nodes - StaticNodes []string `hcl:"static-nodes,optional"` + StaticNodes []string `hcl:"static-nodes,optional" toml:"static-nodes,optional"` // TrustedNodes is the list of trusted nodes - TrustedNodes []string `hcl:"trusted-nodes,optional"` + TrustedNodes []string `hcl:"trusted-nodes,optional" toml:"trusted-nodes,optional"` // DNS is the list of enrtree:// URLs which will be queried for nodes to connect to - DNS []string `hcl:"dns,optional"` + DNS []string `hcl:"dns,optional" toml:"dns,optional"` } type HeimdallConfig struct { // URL is the url of the heimdall server - URL string `hcl:"url,optional"` + URL string `hcl:"url,optional" toml:"url,optional"` // Without is used to disable remote heimdall during testing - Without bool `hcl:"without,optional"` + Without bool `hcl:"bor.without,optional" toml:"bor.without,optional"` } type TxPoolConfig struct { // Locals are the addresses that should be treated by default as local - Locals []string `hcl:"locals,optional"` + Locals []string `hcl:"locals,optional" toml:"locals,optional"` // NoLocals enables whether local transaction handling should be disabled - NoLocals bool `hcl:"no-locals,optional"` + NoLocals bool `hcl:"nolocals,optional" toml:"nolocals,optional"` // Journal is the path to store local transactions to survive node restarts - Journal string `hcl:"journal,optional"` + Journal string `hcl:"journal,optional" toml:"journal,optional"` // Rejournal is the time interval to regenerate the local transaction journal - Rejournal time.Duration - RejournalRaw string `hcl:"rejournal,optional"` + Rejournal time.Duration `hcl:"-,optional" toml:"-,optional"` + RejournalRaw string `hcl:"rejournal,optional" toml:"rejournal,optional"` // PriceLimit is the minimum gas price to enforce for acceptance into the pool - PriceLimit uint64 `hcl:"price-limit,optional"` + PriceLimit uint64 `hcl:"pricelimit,optional" toml:"pricelimit,optional"` // PriceBump is the minimum price bump percentage to replace an already existing transaction (nonce) - PriceBump uint64 `hcl:"price-bump,optional"` + PriceBump uint64 `hcl:"pricebump,optional" toml:"pricebump,optional"` // AccountSlots is the number of executable transaction slots guaranteed per account - AccountSlots uint64 `hcl:"account-slots,optional"` + AccountSlots uint64 `hcl:"accountslots,optional" toml:"accountslots,optional"` // GlobalSlots is the maximum number of executable transaction slots for all accounts - GlobalSlots uint64 `hcl:"global-slots,optional"` + GlobalSlots uint64 `hcl:"globalslots,optional" toml:"globalslots,optional"` // AccountQueue is the maximum number of non-executable transaction slots permitted per account - AccountQueue uint64 `hcl:"account-queue,optional"` + AccountQueue uint64 `hcl:"accountqueue,optional" toml:"accountqueue,optional"` // GlobalQueueis the maximum number of non-executable transaction slots for all accounts - GlobalQueue uint64 `hcl:"global-queue,optional"` + GlobalQueue uint64 `hcl:"globalqueue,optional" toml:"globalqueue,optional"` - // Lifetime is the maximum amount of time non-executable transaction are queued - LifeTime time.Duration - LifeTimeRaw string `hcl:"lifetime,optional"` + // lifetime is the maximum amount of time non-executable transaction are queued + LifeTime time.Duration `hcl:"-,optional" toml:"-,optional"` + LifeTimeRaw string `hcl:"lifetime,optional" toml:"lifetime,optional"` } type SealerConfig struct { // Enabled is used to enable validator mode - Enabled bool `hcl:"enabled,optional"` + Enabled bool `hcl:"mine,optional" toml:"mine,optional"` // Etherbase is the address of the validator - Etherbase string `hcl:"etherbase,optional"` + Etherbase string `hcl:"etherbase,optional" toml:"etherbase,optional"` // ExtraData is the block extra data set by the miner - ExtraData string `hcl:"extra-data,optional"` + ExtraData string `hcl:"extradata,optional" toml:"extradata,optional"` // GasCeil is the target gas ceiling for mined blocks. - GasCeil uint64 `hcl:"gas-ceil,optional"` + GasCeil uint64 `hcl:"gaslimit,optional" toml:"gaslimit,optional"` // GasPrice is the minimum gas price for mining a transaction - GasPrice *big.Int - GasPriceRaw string `hcl:"gas-price,optional"` + GasPrice *big.Int `hcl:"-,optional" toml:"-,optional"` + GasPriceRaw string `hcl:"gasprice,optional" toml:"gasprice,optional"` } type JsonRPCConfig struct { // IPCDisable enables whether ipc is enabled or not - IPCDisable bool `hcl:"ipc-disable,optional"` + IPCDisable bool `hcl:"ipcdisable,optional" toml:"ipcdisable,optional"` // IPCPath is the path of the ipc endpoint - IPCPath string `hcl:"ipc-path,optional"` + IPCPath string `hcl:"ipcpath,optional" toml:"ipcpath,optional"` // GasCap is the global gas cap for eth-call variants. - GasCap uint64 `hcl:"gas-cap,optional"` + GasCap uint64 `hcl:"gascap,optional" toml:"gascap,optional"` // TxFeeCap is the global transaction fee cap for send-transaction variants - TxFeeCap float64 `hcl:"tx-fee-cap,optional"` + TxFeeCap float64 `hcl:"txfeecap,optional" toml:"txfeecap,optional"` // Http has the json-rpc http related settings - Http *APIConfig `hcl:"http,block"` + Http *APIConfig `hcl:"http,block" toml:"http,block"` // Ws has the json-rpc websocket related settings - Ws *APIConfig `hcl:"ws,block"` + Ws *APIConfig `hcl:"ws,block" toml:"ws,block"` // Graphql has the json-rpc graphql related settings - Graphql *APIConfig `hcl:"graphql,block"` + Graphql *APIConfig `hcl:"graphql,block" toml:"graphql,block"` } type GRPCConfig struct { // Addr is the bind address for the grpc rpc server - Addr string + Addr string `hcl:"addr,optional" toml:"addr,optional"` } type APIConfig struct { // Enabled selects whether the api is enabled - Enabled bool `hcl:"enabled,optional"` + Enabled bool `hcl:"enabled,optional" toml:"enabled,optional"` // Port is the port number for this api - Port uint64 `hcl:"port,optional"` + Port uint64 `hcl:"port,optional" toml:"port,optional"` // Prefix is the http prefix to expose this api - Prefix string `hcl:"prefix,optional"` + Prefix string `hcl:"prefix,optional" toml:"prefix,optional"` // Host is the address to bind the api - Host string `hcl:"host,optional"` + Host string `hcl:"host,optional" toml:"host,optional"` - // Modules is the list of enabled api modules - API []string `hcl:"modules,optional"` + // API is the list of enabled api modules + API []string `hcl:"api,optional" toml:"api,optional"` // VHost is the list of valid virtual hosts - VHost []string `hcl:"vhost,optional"` + VHost []string `hcl:"vhosts,optional" toml:"vhosts,optional"` // Cors is the list of Cors endpoints - Cors []string `hcl:"cors,optional"` + Cors []string `hcl:"corsdomain,optional" toml:"corsdomain,optional"` } type GpoConfig struct { // Blocks is the number of blocks to track to compute the price oracle - Blocks uint64 `hcl:"blocks,optional"` + Blocks uint64 `hcl:"blocks,optional" toml:"blocks,optional"` // Percentile sets the weights to new blocks - Percentile uint64 `hcl:"percentile,optional"` + Percentile uint64 `hcl:"percentile,optional" toml:"percentile,optional"` // MaxPrice is an upper bound gas price - MaxPrice *big.Int - MaxPriceRaw string `hcl:"max-price,optional"` + MaxPrice *big.Int `hcl:"-,optional" toml:"-,optional"` + MaxPriceRaw string `hcl:"maxprice,optional" toml:"maxprice,optional"` // IgnorePrice is a lower bound gas price - IgnorePrice *big.Int - IgnorePriceRaw string `hcl:"ignore-price,optional"` + IgnorePrice *big.Int `hcl:"-,optional" toml:"-,optional"` + IgnorePriceRaw string `hcl:"ignoreprice,optional" toml:"ignoreprice,optional"` } type TelemetryConfig struct { // Enabled enables metrics - Enabled bool `hcl:"enabled,optional"` + Enabled bool `hcl:"metrics,optional" toml:"metrics,optional"` // Expensive enables expensive metrics - Expensive bool `hcl:"expensive,optional"` + Expensive bool `hcl:"expensive,optional" toml:"expensive,optional"` // InfluxDB has the influxdb related settings - InfluxDB *InfluxDBConfig `hcl:"influx,block"` + InfluxDB *InfluxDBConfig `hcl:"influx,block" toml:"influx,block"` // Prometheus Address - PrometheusAddr string `hcl:"prometheus-addr,optional"` + PrometheusAddr string `hcl:"prometheus-addr,optional" toml:"prometheus-addr,optional"` // Open collector endpoint - OpenCollectorEndpoint string `hcl:"opencollector-endpoint,optional"` + OpenCollectorEndpoint string `hcl:"opencollector-endpoint,optional" toml:"opencollector-endpoint,optional"` } type InfluxDBConfig struct { // V1Enabled enables influx v1 mode - V1Enabled bool `hcl:"v1-enabled,optional"` + V1Enabled bool `hcl:"influxdb,optional" toml:"influxdb,optional"` // Endpoint is the url endpoint of the influxdb service - Endpoint string `hcl:"endpoint,optional"` + Endpoint string `hcl:"endpoint,optional" toml:"endpoint,optional"` // Database is the name of the database in Influxdb to store the metrics. - Database string `hcl:"database,optional"` + Database string `hcl:"database,optional" toml:"database,optional"` // Enabled is the username to authorize access to Influxdb - Username string `hcl:"username,optional"` + Username string `hcl:"username,optional" toml:"username,optional"` // Password is the password to authorize access to Influxdb - Password string `hcl:"password,optional"` + Password string `hcl:"password,optional" toml:"password,optional"` // Tags are tags attaches to all generated metrics - Tags map[string]string `hcl:"tags,optional"` + Tags map[string]string `hcl:"tags,optional" toml:"tags,optional"` // Enabled enables influx v2 mode - V2Enabled bool `hcl:"v2-enabled,optional"` + V2Enabled bool `hcl:"influxdbv2,optional" toml:"influxdbv2,optional"` // Token is the token to authorize access to Influxdb V2. - Token string `hcl:"token,optional"` + Token string `hcl:"token,optional" toml:"token,optional"` // Bucket is the bucket to store metrics in Influxdb V2. - Bucket string `hcl:"bucket,optional"` + Bucket string `hcl:"bucket,optional" toml:"bucket,optional"` // Organization is the name of the organization for Influxdb V2. - Organization string `hcl:"organization,optional"` + Organization string `hcl:"organization,optional" toml:"organization,optional"` } type CacheConfig struct { // Cache is the amount of cache of the node - Cache uint64 `hcl:"cache,optional"` + Cache uint64 `hcl:"cache,optional" toml:"cache,optional"` // PercGc is percentage of cache used for garbage collection - PercGc uint64 `hcl:"perc-gc,optional"` + PercGc uint64 `hcl:"gc,optional" toml:"gc,optional"` // PercSnapshot is percentage of cache used for snapshots - PercSnapshot uint64 `hcl:"perc-snapshot,optional"` + PercSnapshot uint64 `hcl:"snapshot,optional" toml:"snapshot,optional"` // PercDatabase is percentage of cache used for the database - PercDatabase uint64 `hcl:"perc-database,optional"` + PercDatabase uint64 `hcl:"database,optional" toml:"database,optional"` // PercTrie is percentage of cache used for the trie - PercTrie uint64 `hcl:"perc-trie,optional"` + PercTrie uint64 `hcl:"trie,optional" toml:"trie,optional"` // Journal is the disk journal directory for trie cache to survive node restarts - Journal string `hcl:"journal,optional"` + Journal string `hcl:"journal,optional" toml:"journal,optional"` // Rejournal is the time interval to regenerate the journal for clean cache - Rejournal time.Duration - RejournalRaw string `hcl:"rejournal,optional"` + Rejournal time.Duration `hcl:"-,optional" toml:"-,optional"` + RejournalRaw string `hcl:"rejournal,optional" toml:"rejournal,optional"` // NoPrefetch is used to disable prefetch of tries - NoPrefetch bool `hcl:"no-prefetch,optional"` + NoPrefetch bool `hcl:"noprefetch,optional" toml:"noprefetch,optional"` // Preimages is used to enable the track of hash preimages - Preimages bool `hcl:"preimages,optional"` + Preimages bool `hcl:"preimages,optional" toml:"preimages,optional"` // TxLookupLimit sets the maximum number of blocks from head whose tx indices are reserved. - TxLookupLimit uint64 `hcl:"tx-lookup-limit,optional"` + TxLookupLimit uint64 `hcl:"txlookuplimit,optional" toml:"txlookuplimit,optional"` } type AccountsConfig struct { // Unlock is the list of addresses to unlock in the node - Unlock []string `hcl:"unlock,optional"` + Unlock []string `hcl:"unlock,optional" toml:"unlock,optional"` // PasswordFile is the file where the account passwords are stored - PasswordFile string `hcl:"password-file,optional"` + PasswordFile string `hcl:"password,optional" toml:"password,optional"` // AllowInsecureUnlock allows user to unlock accounts in unsafe http environment. - AllowInsecureUnlock bool `hcl:"allow-insecure-unlock,optional"` + AllowInsecureUnlock bool `hcl:"allow-insecure-unlock,optional" toml:"allow-insecure-unlock,optional"` // UseLightweightKDF enables a faster but less secure encryption of accounts - UseLightweightKDF bool `hcl:"use-lightweight-kdf,optional"` + UseLightweightKDF bool `hcl:"lightkdf,optional" toml:"lightkdf,optional"` // DisableBorWallet disables the personal wallet endpoints - DisableBorWallet bool `hcl:"disable-bor-wallet,optional"` + DisableBorWallet bool `hcl:"disable-bor-wallet,optional" toml:"disable-bor-wallet,optional"` } type DeveloperConfig struct { // Enabled enables the developer mode - Enabled bool `hcl:"dev,optional"` + Enabled bool `hcl:"dev,optional" toml:"dev,optional"` // Period is the block period to use in developer mode - Period uint64 `hcl:"period,optional"` + Period uint64 `hcl:"period,optional" toml:"period,optional"` } func DefaultConfig() *Config { @@ -419,14 +420,14 @@ func DefaultConfig() *Config { Locals: []string{}, NoLocals: false, Journal: "", - Rejournal: time.Duration(1 * time.Hour), + Rejournal: 1 * time.Hour, PriceLimit: 30000000000, PriceBump: 10, AccountSlots: 16, GlobalSlots: 32768, AccountQueue: 16, GlobalQueue: 32768, - LifeTime: time.Duration(3 * time.Hour), + LifeTime: 3 * time.Hour, }, Sealer: &SealerConfig{ Enabled: false, @@ -526,7 +527,7 @@ func (c *Config) fillBigInt() error { }{ {"gpo.maxprice", &c.Gpo.MaxPrice, &c.Gpo.MaxPriceRaw}, {"gpo.ignoreprice", &c.Gpo.IgnorePrice, &c.Gpo.IgnorePriceRaw}, - {"sealer.gasprice", &c.Sealer.GasPrice, &c.Sealer.GasPriceRaw}, + {"miner.gasprice", &c.Sealer.GasPrice, &c.Sealer.GasPriceRaw}, } for _, x := range tds { @@ -582,13 +583,7 @@ func (c *Config) fillTimeDurations() error { func readConfigFile(path string) (*Config, error) { ext := filepath.Ext(path) if ext == ".toml" { - // read file and apply the legacy config - data, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - return readLegacyConfig(data) + return readLegacyConfig(path) } config := &Config{ @@ -1076,7 +1071,7 @@ func Hostname() string { func MakePasswordListFromFile(path string) ([]string, error) { text, err := ioutil.ReadFile(path) if err != nil { - return nil, fmt.Errorf("Failed to read password file: %v", err) + return nil, fmt.Errorf("failed to read password file: %v", err) } lines := strings.Split(string(text), "\n") diff --git a/internal/cli/server/config_legacy.go b/internal/cli/server/config_legacy.go index 0d96b2e023..50508a58b6 100644 --- a/internal/cli/server/config_legacy.go +++ b/internal/cli/server/config_legacy.go @@ -1,33 +1,33 @@ package server import ( - "bytes" + "fmt" + "io/ioutil" - "github.com/naoina/toml" + "github.com/BurntSushi/toml" ) -type legacyConfig struct { - Node struct { - P2P struct { - StaticNodes []string - TrustedNodes []string - } +func readLegacyConfig(path string) (*Config, error) { + data, err := ioutil.ReadFile(path) + tomlData := string(data) + + if err != nil { + return nil, fmt.Errorf("failed to read toml config file: %v", err) } -} -func (l *legacyConfig) Config() *Config { - c := DefaultConfig() - c.P2P.Discovery.StaticNodes = l.Node.P2P.StaticNodes - c.P2P.Discovery.TrustedNodes = l.Node.P2P.TrustedNodes - return c -} + var conf Config -func readLegacyConfig(data []byte) (*Config, error) { - var legacy legacyConfig + if _, err := toml.Decode(tomlData, &conf); err != nil { + return nil, fmt.Errorf("failed to decode toml config file: %v", err) + } - r := toml.NewDecoder(bytes.NewReader(data)) - if err := r.Decode(&legacy); err != nil { + if err := conf.fillBigInt(); err != nil { return nil, err } - return legacy.Config(), nil + + if err := conf.fillTimeDurations(); err != nil { + return nil, err + } + + return &conf, nil } diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index 399481fc9b..6fb662d62b 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -1,21 +1,50 @@ package server import ( + "math/big" "testing" + "time" "github.com/stretchr/testify/assert" ) func TestConfigLegacy(t *testing.T) { - toml := `[Node.P2P] -StaticNodes = ["node1"] -TrustedNodes = ["node2"]` - config, err := readLegacyConfig([]byte(toml)) - if err != nil { - t.Fatal(err) + readFile := func(path string) { + config, err := readLegacyConfig(path) + assert.NoError(t, err) + + assert.Equal(t, config, &Config{ + DataDir: "./data", + RequiredBlocks: map[string]string{ + "a": "b", + }, + P2P: &P2PConfig{ + MaxPeers: 30, + }, + TxPool: &TxPoolConfig{ + Locals: []string{}, + Rejournal: 1 * time.Hour, + LifeTime: 1 * time.Second, + }, + Gpo: &GpoConfig{ + MaxPrice: big.NewInt(100), + IgnorePrice: big.NewInt(2), + }, + Sealer: &SealerConfig{ + Enabled: false, + GasCeil: 20000000, + GasPrice: big.NewInt(30000000000), + }, + Cache: &CacheConfig{ + Cache: 1024, + Rejournal: 1 * time.Hour, + }, + }) } - assert.Equal(t, config.P2P.Discovery.StaticNodes, []string{"node1"}) - assert.Equal(t, config.P2P.Discovery.TrustedNodes, []string{"node2"}) + // read file in hcl format + t.Run("toml", func(t *testing.T) { + readFile("./testdata/test.toml") + }) } diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index c1c6c4ef4a..bf7787ac2e 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -115,7 +115,7 @@ func TestConfigLoadFile(t *testing.T) { MaxPeers: 30, }, TxPool: &TxPoolConfig{ - LifeTime: time.Duration(1 * time.Second), + LifeTime: 1 * time.Second, }, Gpo: &GpoConfig{ MaxPrice: big.NewInt(100), @@ -127,12 +127,12 @@ func TestConfigLoadFile(t *testing.T) { // read file in hcl format t.Run("hcl", func(t *testing.T) { - readFile("./testdata/simple.hcl") + readFile("./testdata/test.hcl") }) // read file in json format t.Run("json", func(t *testing.T) { - readFile("./testdata/simple.json") + readFile("./testdata/test.json") }) } diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index e85428b9ee..bbc7f44036 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -38,7 +38,7 @@ func (c *Command) Flags() *flagset.Flagset { Usage: "Path of the directory to store keystores", Value: &c.cliConfig.KeyStoreDir, }) - f.SliceStringFlag(&flagset.SliceStringFlag{ + f.StringFlag(&flagset.StringFlag{ Name: "config", Usage: "File for the config file", Value: &c.configFile, diff --git a/internal/cli/server/proto/server.pb.go b/internal/cli/server/proto/server.pb.go index 5512d83b72..3e928ac170 100644 --- a/internal/cli/server/proto/server.pb.go +++ b/internal/cli/server/proto/server.pb.go @@ -7,11 +7,12 @@ package proto import ( + reflect "reflect" + sync "sync" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" emptypb "google.golang.org/protobuf/types/known/emptypb" - reflect "reflect" - sync "sync" ) const ( diff --git a/internal/cli/server/proto/server_grpc.pb.go b/internal/cli/server/proto/server_grpc.pb.go index 63f1fa6187..bd4ecb660d 100644 --- a/internal/cli/server/proto/server_grpc.pb.go +++ b/internal/cli/server/proto/server_grpc.pb.go @@ -8,6 +8,7 @@ package proto import ( context "context" + grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" diff --git a/internal/cli/server/service_test.go b/internal/cli/server/service_test.go index 7850525686..86cf68e75e 100644 --- a/internal/cli/server/service_test.go +++ b/internal/cli/server/service_test.go @@ -4,8 +4,9 @@ import ( "math/big" "testing" - "github.com/ethereum/go-ethereum/internal/cli/server/proto" "github.com/stretchr/testify/assert" + + "github.com/ethereum/go-ethereum/internal/cli/server/proto" ) func TestGatherBlocks(t *testing.T) { diff --git a/internal/cli/server/testdata/simple.json b/internal/cli/server/testdata/simple.json deleted file mode 100644 index 6270ee6d13..0000000000 --- a/internal/cli/server/testdata/simple.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "data-dir": "./data", - "requiredblocks": { - "a": "b" - }, - "p2p": { - "max-peers": 30 - }, - "txpool": { - "lifetime": "1s" - }, - "gpo": { - "max-price": "100" - } -} \ No newline at end of file diff --git a/internal/cli/server/testdata/simple.hcl b/internal/cli/server/testdata/test.hcl similarity index 57% rename from internal/cli/server/testdata/simple.hcl rename to internal/cli/server/testdata/test.hcl index 5afc091859..208fdc51c1 100644 --- a/internal/cli/server/testdata/simple.hcl +++ b/internal/cli/server/testdata/test.hcl @@ -1,11 +1,11 @@ -data-dir = "./data" +datadir = "./data" requiredblocks = { a = "b" } p2p { - max-peers = 30 + maxpeers = 30 } txpool { @@ -13,5 +13,5 @@ txpool { } gpo { - max-price = "100" -} + maxprice = "100" +} \ No newline at end of file diff --git a/internal/cli/server/testdata/test.json b/internal/cli/server/testdata/test.json new file mode 100644 index 0000000000..467835e755 --- /dev/null +++ b/internal/cli/server/testdata/test.json @@ -0,0 +1,15 @@ +{ + "datadir": "./data", + "requiredblocks": { + "a": "b" + }, + "p2p": { + "maxpeers": 30 + }, + "txpool": { + "lifetime": "1s" + }, + "gpo": { + "maxprice": "100" + } +} \ No newline at end of file diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml new file mode 100644 index 0000000000..81fc4c4630 --- /dev/null +++ b/internal/cli/server/testdata/test.toml @@ -0,0 +1,25 @@ +datadir = "./data" + +[requiredblocks] +a = "b" + +[p2p] +maxpeers = 30 + +[txpool] +locals = [] +rejournal = "1h0m0s" +lifetime = "1s" + +[miner] +mine = false +gaslimit = 20000000 +gasprice = "30000000000" + +[gpo] +maxprice = "100" +ignoreprice = "2" + +[cache] +cache = 1024 +rejournal = "1h0m0s" From 8378cc9ecb9159b7a35027eca22aa273b67b0fb3 Mon Sep 17 00:00:00 2001 From: Jerry Date: Thu, 28 Jul 2022 12:33:23 -0700 Subject: [PATCH 03/70] Add 'bor' user during package installation --- .goreleaser.yml | 9 +++++++++ builder/files/bor-post-install.sh | 9 +++++++++ builder/files/bor.service | 2 +- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 builder/files/bor-post-install.sh diff --git a/.goreleaser.yml b/.goreleaser.yml index 7fbc61ca4a..acafc4abc0 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -75,12 +75,18 @@ nfpms: description: Polygon Blockchain license: GPLv3 LGPLv3 + bindir: /usr/local/bin + formats: - apk - deb - rpm contents: + - dst: /var/lib/bor + type: dir + file_info: + mode: 0777 - src: builder/files/bor.service dst: /lib/systemd/system/bor.service type: config @@ -94,6 +100,9 @@ nfpms: dst: /var/lib/bor/config.toml type: config + scripts: + postinstall: builder/files/bor-post-install.sh + overrides: rpm: replacements: diff --git a/builder/files/bor-post-install.sh b/builder/files/bor-post-install.sh new file mode 100644 index 0000000000..1419479983 --- /dev/null +++ b/builder/files/bor-post-install.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -e + +PKG="bor" + +if ! getent passwd $PKG >/dev/null ; then + adduser --disabled-password --disabled-login --shell /usr/sbin/nologin --quiet --system --no-create-home --home /nonexistent $PKG + echo "Created system user $PKG" +fi diff --git a/builder/files/bor.service b/builder/files/bor.service index 4b628cbf6e..2deff3dbc9 100644 --- a/builder/files/bor.service +++ b/builder/files/bor.service @@ -8,7 +8,7 @@ RestartSec=5s ExecStart=/usr/local/bin/bor server -config="/var/lib/bor/config.toml" Type=simple - User=ubuntu + User=bor KillSignal=SIGINT TimeoutStopSec=120 From 694dc02a0196d947266355b6560a5c5004aac9e8 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 29 Jul 2022 17:30:25 +0530 Subject: [PATCH 04/70] fixed the bug caused by fillBigInt and FillTime.Duration functions (#474) --- internal/cli/server/config_legacy.go | 2 +- internal/cli/server/config_legacy_test.go | 130 ++++++++++++++++++++-- 2 files changed, 120 insertions(+), 12 deletions(-) diff --git a/internal/cli/server/config_legacy.go b/internal/cli/server/config_legacy.go index 50508a58b6..9411b8290d 100644 --- a/internal/cli/server/config_legacy.go +++ b/internal/cli/server/config_legacy.go @@ -15,7 +15,7 @@ func readLegacyConfig(path string) (*Config, error) { return nil, fmt.Errorf("failed to read toml config file: %v", err) } - var conf Config + conf := *DefaultConfig() if _, err := toml.Decode(tomlData, &conf); err != nil { return nil, fmt.Errorf("failed to decode toml config file: %v", err) diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index 6fb662d62b..1b52096216 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -6,6 +6,8 @@ import ( "time" "github.com/stretchr/testify/assert" + + "github.com/ethereum/go-ethereum/eth/ethconfig" ) func TestConfigLegacy(t *testing.T) { @@ -15,30 +17,136 @@ func TestConfigLegacy(t *testing.T) { assert.NoError(t, err) assert.Equal(t, config, &Config{ - DataDir: "./data", + Chain: "mainnet", + Identity: Hostname(), RequiredBlocks: map[string]string{ "a": "b", }, + LogLevel: "INFO", + DataDir: "./data", P2P: &P2PConfig{ - MaxPeers: 30, + MaxPeers: 30, + MaxPendPeers: 50, + Bind: "0.0.0.0", + Port: 30303, + NoDiscover: false, + NAT: "any", + Discovery: &P2PDiscovery{ + V5Enabled: false, + Bootnodes: []string{}, + BootnodesV4: []string{}, + BootnodesV5: []string{}, + StaticNodes: []string{}, + TrustedNodes: []string{}, + DNS: []string{}, + }, }, + Heimdall: &HeimdallConfig{ + URL: "http://localhost:1317", + Without: false, + }, + SyncMode: "full", + GcMode: "full", + Snapshot: true, TxPool: &TxPoolConfig{ - Locals: []string{}, - Rejournal: 1 * time.Hour, - LifeTime: 1 * time.Second, + Locals: []string{}, + NoLocals: false, + Journal: "", + Rejournal: 1 * time.Hour, + PriceLimit: 30000000000, + PriceBump: 10, + AccountSlots: 16, + GlobalSlots: 32768, + AccountQueue: 16, + GlobalQueue: 32768, + LifeTime: 1 * time.Second, + }, + Sealer: &SealerConfig{ + Enabled: false, + Etherbase: "", + GasCeil: 20000000, + GasPrice: big.NewInt(30000000000), + ExtraData: "", }, Gpo: &GpoConfig{ + Blocks: 20, + Percentile: 60, MaxPrice: big.NewInt(100), IgnorePrice: big.NewInt(2), }, - Sealer: &SealerConfig{ - Enabled: false, - GasCeil: 20000000, - GasPrice: big.NewInt(30000000000), + JsonRPC: &JsonRPCConfig{ + IPCDisable: false, + IPCPath: "", + GasCap: ethconfig.Defaults.RPCGasCap, + TxFeeCap: ethconfig.Defaults.RPCTxFeeCap, + Http: &APIConfig{ + Enabled: false, + Port: 8545, + Prefix: "", + Host: "localhost", + API: []string{"eth", "net", "web3", "txpool", "bor"}, + Cors: []string{"*"}, + VHost: []string{"*"}, + }, + Ws: &APIConfig{ + Enabled: false, + Port: 8546, + Prefix: "", + Host: "localhost", + API: []string{"web3", "net"}, + Cors: []string{"*"}, + VHost: []string{"*"}, + }, + Graphql: &APIConfig{ + Enabled: false, + Cors: []string{"*"}, + VHost: []string{"*"}, + }, + }, + Ethstats: "", + Telemetry: &TelemetryConfig{ + Enabled: false, + Expensive: false, + PrometheusAddr: "", + OpenCollectorEndpoint: "", + InfluxDB: &InfluxDBConfig{ + V1Enabled: false, + Endpoint: "", + Database: "", + Username: "", + Password: "", + Tags: map[string]string{}, + V2Enabled: false, + Token: "", + Bucket: "", + Organization: "", + }, }, Cache: &CacheConfig{ - Cache: 1024, - Rejournal: 1 * time.Hour, + Cache: 1024, + PercDatabase: 50, + PercTrie: 15, + PercGc: 25, + PercSnapshot: 10, + Journal: "triecache", + Rejournal: 1 * time.Hour, + NoPrefetch: false, + Preimages: false, + TxLookupLimit: 2350000, + }, + Accounts: &AccountsConfig{ + Unlock: []string{}, + PasswordFile: "", + AllowInsecureUnlock: false, + UseLightweightKDF: false, + DisableBorWallet: false, + }, + GRPC: &GRPCConfig{ + Addr: ":3131", + }, + Developer: &DeveloperConfig{ + Enabled: false, + Period: 0, }, }) } From 18fafc01c673c4996748072d804981f13953fb88 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 4 Aug 2022 12:42:50 +0530 Subject: [PATCH 05/70] disable bor wallet by default --- internal/cli/server/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 819ab1aca5..34ad0e60f7 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -507,7 +507,7 @@ func DefaultConfig() *Config { PasswordFile: "", AllowInsecureUnlock: false, UseLightweightKDF: false, - DisableBorWallet: false, + DisableBorWallet: true, }, GRPC: &GRPCConfig{ Addr: ":3131", From ba184918413bac197a53914693fdd0728cfd182a Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 4 Aug 2022 16:33:30 +0530 Subject: [PATCH 06/70] fix: tests --- internal/cli/server/config_legacy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index 1b52096216..a8127915a1 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -139,7 +139,7 @@ func TestConfigLegacy(t *testing.T) { PasswordFile: "", AllowInsecureUnlock: false, UseLightweightKDF: false, - DisableBorWallet: false, + DisableBorWallet: true, }, GRPC: &GRPCConfig{ Addr: ":3131", From 49cd8c8e7c10e3d0d0050c0e0fd3499c059eee81 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 17 Aug 2022 13:28:55 +0530 Subject: [PATCH 07/70] update version --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 4b608b1a5f..ec3923a884 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 2 // Minor version component of the current release - VersionPatch = 16 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 2 // Minor version component of the current release + VersionPatch = 17 // Patch version component of the current release + VersionMeta = "beta" // Version metadata to append to the version string ) // Version holds the textual version string. From 43b67c361f02d97f3377d01b7b42bd013cac6ca7 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 17 Aug 2022 13:55:27 +0530 Subject: [PATCH 08/70] handle snap sync mode switches (#495) --- cmd/utils/flags.go | 7 +++++++ eth/handler.go | 11 +++++++++-- eth/sync.go | 43 +++++++++++++++++++++++++++---------------- 3 files changed, 43 insertions(+), 18 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e2bbdb17f8..79c73903fb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1620,6 +1620,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) + + // To be extra preventive, we won't allow the node to start + // in snap sync mode until we have it working + // TODO(snap): Comment when we have snap sync working + if cfg.SyncMode == downloader.SnapSync { + cfg.SyncMode = downloader.FullSync + } } if ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) diff --git a/eth/handler.go b/eth/handler.go index 40edfa2d17..8d56537df9 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -155,8 +155,15 @@ func newHandler(config *handlerConfig) (*handler, error) { // In these cases however it's safe to reenable snap sync. fullBlock, fastBlock := h.chain.CurrentBlock(), h.chain.CurrentFastBlock() if fullBlock.NumberU64() == 0 && fastBlock.NumberU64() > 0 { - h.snapSync = uint32(1) - log.Warn("Switch sync mode from full sync to snap sync") + // Note: Ideally this should never happen with bor, but to be extra + // preventive we won't allow it to roll over to snap sync until + // we have it working + + // TODO(snap): Uncomment when we have snap sync working + // h.snapSync = uint32(1) + // log.Warn("Switch sync mode from full sync to snap sync") + + log.Warn("Preventing switching sync mode from full sync to snap sync") } } else { if h.chain.CurrentBlock().NumberU64() > 0 { diff --git a/eth/sync.go b/eth/sync.go index d67d2311d0..aa79b6181c 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -204,25 +204,36 @@ func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { } func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { - // If we're in snap sync mode, return that directly - if atomic.LoadUint32(&cs.handler.snapSync) == 1 { - block := cs.handler.chain.CurrentFastBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) - return downloader.SnapSync, td - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should reenable - if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { - if head := cs.handler.chain.CurrentBlock(); head.NumberU64() < *pivot { - block := cs.handler.chain.CurrentFastBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) - return downloader.SnapSync, td - } - } - // Nope, we're really full syncing + // Note: Ideally this should never happen with bor, but to be extra + // preventive we won't allow it to roll over to snap sync until + // we have it working + + // Handle full sync mode only head := cs.handler.chain.CurrentBlock() td := cs.handler.chain.GetTd(head.Hash(), head.NumberU64()) return downloader.FullSync, td + + // TODO(snap): Uncomment when we have snap sync working + + // If we're in snap sync mode, return that directly + // if atomic.LoadUint32(&cs.handler.snapSync) == 1 { + // block := cs.handler.chain.CurrentFastBlock() + // td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) + // return downloader.SnapSync, td + // } + // // We are probably in full sync, but we might have rewound to before the + // // snap sync pivot, check if we should reenable + // if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { + // if head := cs.handler.chain.CurrentBlock(); head.NumberU64() < *pivot { + // block := cs.handler.chain.CurrentFastBlock() + // td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) + // return downloader.SnapSync, td + // } + // } + // // Nope, we're really full syncing + // head := cs.handler.chain.CurrentBlock() + // td := cs.handler.chain.GetTd(head.Hash(), head.NumberU64()) + // return downloader.FullSync, td } // startSync launches doSync in a new goroutine. From 77f444a24b78f1c77cb4d7b95c5011705478441f Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 17 Aug 2022 21:11:40 +0530 Subject: [PATCH 09/70] fix: whitelist/requiredblocks regression (#497) --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 8d56537df9..bd31e0f117 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -432,7 +432,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { }() } // If we have any explicit peer required block hashes, request them - for number := range h.peerRequiredBlocks { + for number, hash := range h.peerRequiredBlocks { resCh := make(chan *eth.Response) if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { return err From 10c1ff2a7434246aba9e6ec3197bd3996dd2c5a8 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 9 Aug 2022 09:16:42 +0530 Subject: [PATCH 10/70] internal/cli: add support for removedb (#478) * internal/cli: add support for removedb * update docs * internal/cli: use constant path, handle err --- docs/cli/README.md | 2 + docs/cli/removedb.md | 7 ++ internal/cli/command.go | 5 ++ internal/cli/removedb.go | 154 ++++++++++++++++++++++++++++++++++ internal/cli/server/config.go | 4 +- 5 files changed, 170 insertions(+), 2 deletions(-) create mode 100644 docs/cli/removedb.md create mode 100644 internal/cli/removedb.go diff --git a/docs/cli/README.md b/docs/cli/README.md index 3bc3daddc5..bf37d6ef56 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -40,6 +40,8 @@ - [```peers status```](./peers_status.md) +- [```removedb```](./removedb.md) + - [```server```](./server.md) - [```status```](./status.md) diff --git a/docs/cli/removedb.md b/docs/cli/removedb.md new file mode 100644 index 0000000000..3c6e84f1d6 --- /dev/null +++ b/docs/cli/removedb.md @@ -0,0 +1,7 @@ +# RemoveDB + +The ```bor removedb``` command will remove the blockchain and state databases at the given datadir location + +## Options + +- ```datadir```: Path of the data directory to store information diff --git a/internal/cli/command.go b/internal/cli/command.go index 06127a9823..93dca4cb3e 100644 --- a/internal/cli/command.go +++ b/internal/cli/command.go @@ -184,6 +184,11 @@ func Commands() map[string]MarkDownCommandFactory { UI: ui, }, nil }, + "removedb": func() (MarkDownCommand, error) { + return &RemoveDBCommand{ + Meta2: meta2, + }, nil + }, } } diff --git a/internal/cli/removedb.go b/internal/cli/removedb.go new file mode 100644 index 0000000000..4a604086ed --- /dev/null +++ b/internal/cli/removedb.go @@ -0,0 +1,154 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/internal/cli/flagset" + "github.com/ethereum/go-ethereum/internal/cli/server" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + + "github.com/mitchellh/cli" +) + +// RemoveDBCommand is for removing blockchain and state databases +type RemoveDBCommand struct { + *Meta2 + + datadir string +} + +const ( + chaindataPath string = "chaindata" + ancientPath string = "ancient" + lightchaindataPath string = "lightchaindata" +) + +// MarkDown implements cli.MarkDown interface +func (c *RemoveDBCommand) MarkDown() string { + items := []string{ + "# RemoveDB", + "The ```bor removedb``` command will remove the blockchain and state databases at the given datadir location", + c.Flags().MarkDown(), + } + + return strings.Join(items, "\n\n") +} + +// Help implements the cli.Command interface +func (c *RemoveDBCommand) Help() string { + return `Usage: bor removedb + + This command will remove the blockchain and state databases at the given datadir location` +} + +// Synopsis implements the cli.Command interface +func (c *RemoveDBCommand) Synopsis() string { + return "Remove blockchain and state databases" +} + +func (c *RemoveDBCommand) Flags() *flagset.Flagset { + flags := c.NewFlagSet("removedb") + + flags.StringFlag(&flagset.StringFlag{ + Name: "datadir", + Value: &c.datadir, + Usage: "Path of the data directory to store information", + }) + + return flags +} + +// Run implements the cli.Command interface +func (c *RemoveDBCommand) Run(args []string) int { + flags := c.Flags() + + // parse datadir + if err := flags.Parse(args); err != nil { + c.UI.Error(err.Error()) + return 1 + } + + datadir := c.datadir + if datadir == "" { + datadir = server.DefaultDataDir() + } + + // create ethereum node config with just the datadir + nodeCfg := &node.Config{DataDir: datadir} + + // Remove the full node state database + path := nodeCfg.ResolvePath(chaindataPath) + if common.FileExist(path) { + confirmAndRemoveDB(c.UI, path, "full node state database") + } else { + log.Info("Full node state database missing", "path", path) + } + + // Remove the full node ancient database + // Note: The old cli used DatabaseFreezer path from config if provided explicitly + // We don't have access to eth config and hence we assume it to be + // under the "chaindata" folder. + path = filepath.Join(nodeCfg.ResolvePath(chaindataPath), ancientPath) + if common.FileExist(path) { + confirmAndRemoveDB(c.UI, path, "full node ancient database") + } else { + log.Info("Full node ancient database missing", "path", path) + } + + // Remove the light node database + path = nodeCfg.ResolvePath(lightchaindataPath) + if common.FileExist(path) { + confirmAndRemoveDB(c.UI, path, "light node database") + } else { + log.Info("Light node database missing", "path", path) + } + + return 0 +} + +// confirmAndRemoveDB prompts the user for a last confirmation and removes the +// folder if accepted. +func confirmAndRemoveDB(ui cli.Ui, database string, kind string) { + for { + confirm, err := ui.Ask(fmt.Sprintf("Remove %s (%s)? [y/n]", kind, database)) + + switch { + case err != nil: + ui.Output(err.Error()) + return + case confirm != "": + switch strings.ToLower(confirm) { + case "y": + start := time.Now() + err = filepath.Walk(database, func(path string, info os.FileInfo, err error) error { + // If we're at the top level folder, recurse into + if path == database { + return nil + } + // Delete all the files, but not subfolders + if !info.IsDir() { + return os.Remove(path) + } + return filepath.SkipDir + }) + + if err != nil && err != filepath.SkipDir { + ui.Output(err.Error()) + } else { + log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start))) + } + + return + case "n": + log.Info("Database deletion skipped", "path", database) + return + } + } + } +} diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 34ad0e60f7..faa8452742 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -391,7 +391,7 @@ func DefaultConfig() *Config { Identity: Hostname(), RequiredBlocks: map[string]string{}, LogLevel: "INFO", - DataDir: defaultDataDir(), + DataDir: DefaultDataDir(), P2P: &P2PConfig{ MaxPeers: 30, MaxPendPeers: 50, @@ -1035,7 +1035,7 @@ func parseBootnodes(urls []string) ([]*enode.Node, error) { return dst, nil } -func defaultDataDir() string { +func DefaultDataDir() string { // Try to place the data folder in the user's home dir home, _ := homedir.Dir() if home == "" { From 0b28230c22b907cad39872de97602b8f2ea39f85 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Wed, 17 Aug 2022 12:01:53 +0530 Subject: [PATCH 11/70] updated config.toml in builder/files, added all fields and commented those which are not used (#491) --- builder/files/config.toml | 161 ++++++++++++++++++++++++++++++-------- 1 file changed, 127 insertions(+), 34 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index c55a143ed7..d8503e4351 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -1,44 +1,137 @@ -# chain = "mumbai" +# NOTE: Uncomment and configure the following 8 fields in case you run a validator: +# `mine`, `etherbase`, `nodiscover`, `maxpeers`, `keystore`, `allow-insecure-unlock`, `password`, `unlock` + chain = "mainnet" +# chain = "mumbai" +# identity = "Pratiks-MacBook-Pro.local" +# log-level = "INFO" datadir = "/var/lib/bor/data" +# keystore = "/var/lib/bor/keystore" syncmode = "full" +# gcmode = "full" +# snapshot = true +# ethstats = "" -[telemetry] -metrics = true -prometheus-addr = "127.0.0.1:7071" - -[miner] -# *** Validator params -# *** Uncomment and configure the following lines in case you run a validator. -# mine = true -# etherbase = "VALIDATOR ADDRESS" -gasprice = "30000000000" -gasceil = 20000000 - - -[txpool] -accountqueue = 16 -accountslots = 16 -globalqueue = 32768 -globalslots = 32768 -lifetime = "1h30m0s" -nolocals = true -pricelimit = 30000000000 +# [requiredblocks] [p2p] -# *** Validator params -# *** Uncomment and configure the following lines in case you run a validator. -# nodiscover = true -# maxpeers = 1 + # maxpeers = 1 + # nodiscover = true + # maxpendpeers = 50 + # bind = "0.0.0.0" + # port = 30303 + # nat = "any" [p2p.discovery] - bootnodes = ["enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303", "enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303"] + # v5disc = false + bootnodes = ["enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303", "enode://88116f4295f5a31538ae409e4d44ad40d22e44ee9342869e7d68bdec55b0f83c1530355ce8b41fbec0928a7d75a5745d528450d30aec92066ab6ba1ee351d710@159.203.9.164:30303"] + # Uncomment below `bootnodes` field for Mumbai bootnode + # bootnodes = ["enode://095c4465fe509bd7107bbf421aea0d3ad4d4bfc3ff8f9fdc86f4f950892ae3bbc3e5c715343c4cf60c1c06e088e621d6f1b43ab9130ae56c2cacfd356a284ee4@18.213.200.99:30303"] + # bootnodesv4 = [] + # bootnodesv5 = [] + # static-nodes = [] + # trusted-nodes = [] + # dns = [] -# *** Validator params -# *** Uncomment and configure the following lines in case you run a validator. +# [heimdall] + # url = "http://localhost:1317" + # "bor.without" = false + # grpc-address = "" -# keystore = "/var/lib/bor/keystore" +[txpool] + nolocals = true + pricelimit = 30000000000 + accountslots = 16 + globalslots = 32768 + accountqueue = 16 + globalqueue = 32768 + lifetime = "1h30m0s" + # locals = [] + # journal = "" + # rejournal = "1h0m0s" + # pricebump = 10 -# [accounts] -# allow-insecure-unlock = true -# password = "/var/lib/bor/password.txt" -# unlock = ["VALIDATOR ADDRESS"] \ No newline at end of file +[miner] + gaslimit = 20000000 + gasprice = "30000000000" + # mine = true + # etherbase = "VALIDATOR ADDRESS" + # extradata = "" + + +# [jsonrpc] + # ipcdisable = false + # ipcpath = "" + # gascap = 50000000 + # txfeecap = 5.0 + # [jsonrpc.http] + # enabled = false + # port = 8545 + # prefix = "" + # host = "localhost" + # api = ["eth", "net", "web3", "txpool", "bor"] + # vhosts = ["*"] + # corsdomain = ["*"] + # [jsonrpc.ws] + # enabled = false + # port = 8546 + # prefix = "" + # host = "localhost" + # api = ["web3", "net"] + # vhosts = ["*"] + # corsdomain = ["*"] + # [jsonrpc.graphql] + # enabled = false + # port = 0 + # prefix = "" + # host = "" + # vhosts = ["*"] + # corsdomain = ["*"] + +# [gpo] + # blocks = 20 + # percentile = 60 + # maxprice = "5000000000000" + # ignoreprice = "2" + +[telemetry] + metrics = true + # expensive = false + prometheus-addr = "127.0.0.1:7071" + # opencollector-endpoint = "" + # [telemetry.influx] + # influxdb = false + # endpoint = "" + # database = "" + # username = "" + # password = "" + # influxdbv2 = false + # token = "" + # bucket = "" + # organization = "" + # [telemetry.influx.tags] + +# [cache] + # cache = 1024 + # gc = 25 + # snapshot = 10 + # database = 50 + # trie = 15 + # journal = "triecache" + # rejournal = "1h0m0s" + # noprefetch = false + # preimages = false + # txlookuplimit = 2350000 + +[accounts] + # allow-insecure-unlock = true + # password = "/var/lib/bor/password.txt" + # unlock = ["VALIDATOR ADDRESS"] + # lightkdf = false + # disable-bor-wallet = false + +# [grpc] + # addr = ":3131" + +# [developer] + # dev = false + # period = 0 \ No newline at end of file From 2aacbde0976c6d12fbd349c394c6337537f7d026 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 17 Aug 2022 15:18:16 +0530 Subject: [PATCH 12/70] eth, cli: prevent snap sync mode migration - v0.3.x (#494) * handle snap sync mode switches * fix linters --- cmd/utils/flags.go | 7 ++++++ eth/handler.go | 11 +++++++-- eth/sync.go | 43 ++++++++++++++++++++++------------- internal/cli/server/config.go | 5 +++- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e2bbdb17f8..79c73903fb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1620,6 +1620,13 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.GlobalIsSet(SyncModeFlag.Name) { cfg.SyncMode = *GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode) + + // To be extra preventive, we won't allow the node to start + // in snap sync mode until we have it working + // TODO(snap): Comment when we have snap sync working + if cfg.SyncMode == downloader.SnapSync { + cfg.SyncMode = downloader.FullSync + } } if ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = ctx.GlobalUint64(NetworkIdFlag.Name) diff --git a/eth/handler.go b/eth/handler.go index ab95f5f769..3d8380412c 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -161,8 +161,15 @@ func newHandler(config *handlerConfig) (*handler, error) { // In these cases however it's safe to reenable snap sync. fullBlock, fastBlock := h.chain.CurrentBlock(), h.chain.CurrentFastBlock() if fullBlock.NumberU64() == 0 && fastBlock.NumberU64() > 0 { - h.snapSync = uint32(1) - log.Warn("Switch sync mode from full sync to snap sync") + // Note: Ideally this should never happen with bor, but to be extra + // preventive we won't allow it to roll over to snap sync until + // we have it working + + // TODO(snap): Uncomment when we have snap sync working + // h.snapSync = uint32(1) + // log.Warn("Switch sync mode from full sync to snap sync") + + log.Warn("Preventing switching sync mode from full sync to snap sync") } } else { if h.chain.CurrentBlock().NumberU64() > 0 { diff --git a/eth/sync.go b/eth/sync.go index d67d2311d0..22c0c9054a 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -204,25 +204,36 @@ func peerToSyncOp(mode downloader.SyncMode, p *eth.Peer) *chainSyncOp { } func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { - // If we're in snap sync mode, return that directly - if atomic.LoadUint32(&cs.handler.snapSync) == 1 { - block := cs.handler.chain.CurrentFastBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) - return downloader.SnapSync, td - } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should reenable - if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { - if head := cs.handler.chain.CurrentBlock(); head.NumberU64() < *pivot { - block := cs.handler.chain.CurrentFastBlock() - td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) - return downloader.SnapSync, td - } - } - // Nope, we're really full syncing + // Note: Ideally this should never happen with bor, but to be extra + // preventive we won't allow it to roll over to snap sync until + // we have it working + + // Handle full sync mode only head := cs.handler.chain.CurrentBlock() td := cs.handler.chain.GetTd(head.Hash(), head.NumberU64()) return downloader.FullSync, td + + // TODO(snap): Uncomment when we have snap sync working + + // If we're in snap sync mode, return that directly + // if atomic.LoadUint32(&cs.handler.snapSync) == 1 { + // block := cs.handler.chain.CurrentFastBlock() + // td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) + // return downloader.SnapSync, td + // } + // // We are probably in full sync, but we might have rewound to before the + // // snap sync pivot, check if we should reenable + // if pivot := rawdb.ReadLastPivotNumber(cs.handler.database); pivot != nil { + // if head := cs.handler.chain.CurrentBlock(); head.NumberU64() < *pivot { + // block := cs.handler.chain.CurrentFastBlock() + // td := cs.handler.chain.GetTd(block.Hash(), block.NumberU64()) + // return downloader.SnapSync, td + // } + // } + // Nope, we're really full syncing + // head := cs.handler.chain.CurrentBlock() + // td := cs.handler.chain.GetTd(head.Hash(), head.NumberU64()) + // return downloader.FullSync, td } // startSync launches doSync in a new goroutine. diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index faa8452742..e10b7e9c36 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -843,7 +843,10 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* case "full": n.SyncMode = downloader.FullSync case "snap": - n.SyncMode = downloader.SnapSync + // n.SyncMode = downloader.SnapSync // TODO(snap): Uncomment when we have snap sync working + n.SyncMode = downloader.FullSync + + log.Warn("Bor doesn't support Snap Sync yet, switching to Full Sync mode") default: return nil, fmt.Errorf("sync mode '%s' not found", c.SyncMode) } From 8c97faaa8c9a046266aa7a0636df3ee55956dbc0 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 5 Aug 2022 14:56:41 +0530 Subject: [PATCH 13/70] internal/cli, cmd/geth: replaced package naoina/toml with BurntSushi/toml and updated config name-tags (#486) * removed package naoina/toml from dumpconfig and added BurntSushi/toml * updated cmd/gethconfig.go --- cmd/geth/config.go | 57 ++++------------------- go.mod | 2 - go.sum | 4 -- internal/cli/dumpconfig.go | 20 ++------ internal/cli/server/config.go | 12 ++--- internal/cli/server/config_legacy_test.go | 7 +-- internal/cli/server/testdata/test.toml | 6 +-- 7 files changed, 25 insertions(+), 83 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index d8ba5366fe..08b76f83da 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -17,17 +17,16 @@ package main import ( - "bufio" - "errors" "fmt" + "io/ioutil" "math/big" "os" - "reflect" "time" - "unicode" "gopkg.in/urfave/cli.v1" + "github.com/BurntSushi/toml" + "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" @@ -41,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" - "github.com/naoina/toml" ) var ( @@ -61,28 +59,6 @@ var ( } ) -// These settings ensure that TOML keys use the same names as Go struct fields. -var tomlSettings = toml.Config{ - NormFieldName: func(rt reflect.Type, key string) string { - return key - }, - FieldToKey: func(rt reflect.Type, field string) string { - return field - }, - MissingField: func(rt reflect.Type, field string) error { - id := fmt.Sprintf("%s.%s", rt.String(), field) - if deprecated(id) { - log.Warn("Config field is deprecated and won't have an effect", "name", id) - return nil - } - var link string - if unicode.IsUpper(rune(rt.Name()[0])) && rt.PkgPath() != "main" { - link = fmt.Sprintf(", see https://godoc.org/%s#%s for available fields", rt.PkgPath(), rt.Name()) - } - return fmt.Errorf("field '%s' is not defined in %s%s", field, rt.String(), link) - }, -} - type ethstatsConfig struct { URL string `toml:",omitempty"` } @@ -95,18 +71,17 @@ type gethConfig struct { } func loadConfig(file string, cfg *gethConfig) error { - f, err := os.Open(file) + data, err := ioutil.ReadFile(file) if err != nil { return err } - defer f.Close() - err = tomlSettings.NewDecoder(bufio.NewReader(f)).Decode(cfg) - // Add file name to errors that have a line number. - if _, ok := err.(*toml.LineError); ok { - err = errors.New(file + ", " + err.Error()) + tomlData := string(data) + if _, err = toml.Decode(tomlData, &cfg); err != nil { + return err } - return err + + return nil } func defaultNodeConfig() node.Config { @@ -214,22 +189,10 @@ func dumpConfig(ctx *cli.Context) error { comment += "# Note: this config doesn't contain the genesis block.\n\n" } - out, err := tomlSettings.Marshal(&cfg) - if err != nil { + if err := toml.NewEncoder(os.Stdout).Encode(&cfg); err != nil { return err } - dump := os.Stdout - if ctx.NArg() > 0 { - dump, err = os.OpenFile(ctx.Args().Get(0), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return err - } - defer dump.Close() - } - dump.WriteString(comment) - dump.Write(out) - return nil } diff --git a/go.mod b/go.mod index 3f90695d47..39bbeb240d 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,6 @@ require ( github.com/mitchellh/cli v1.1.2 github.com/mitchellh/go-grpc-net-conn v0.0.0-20200427190222-eb030e4876f0 github.com/mitchellh/go-homedir v1.1.0 - github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.7.1 @@ -119,7 +118,6 @@ require ( github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect - github.com/naoina/go-stringutil v0.1.0 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index fc58b621e6..88c9152eef 100644 --- a/go.sum +++ b/go.sum @@ -374,10 +374,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= -github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= -github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= -github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= diff --git a/internal/cli/dumpconfig.go b/internal/cli/dumpconfig.go index 3e4688fc24..dad0be923d 100644 --- a/internal/cli/dumpconfig.go +++ b/internal/cli/dumpconfig.go @@ -1,24 +1,14 @@ package cli import ( - "reflect" + "os" "strings" - "github.com/naoina/toml" + "github.com/BurntSushi/toml" "github.com/ethereum/go-ethereum/internal/cli/server" ) -// These settings ensure that TOML keys use the same names as Go struct fields. -var tomlSettings = toml.Config{ - NormFieldName: func(rt reflect.Type, key string) string { - return key - }, - FieldToKey: func(rt reflect.Type, field string) string { - return field - }, -} - // DumpconfigCommand is for exporting user provided flags into a config file type DumpconfigCommand struct { *Meta2 @@ -69,14 +59,10 @@ func (c *DumpconfigCommand) Run(args []string) int { userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String() userConfig.Cache.RejournalRaw = userConfig.Cache.Rejournal.String() - // Currently, the configurations (userConfig) is exported into `toml` file format. - out, err := tomlSettings.Marshal(&userConfig) - if err != nil { + if err := toml.NewEncoder(os.Stdout).Encode(userConfig); err != nil { c.UI.Error(err.Error()) return 1 } - c.UI.Output(string(out)) - return 0 } diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index e10b7e9c36..255cf633eb 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -167,7 +167,7 @@ type TxPoolConfig struct { Journal string `hcl:"journal,optional" toml:"journal,optional"` // Rejournal is the time interval to regenerate the local transaction journal - Rejournal time.Duration `hcl:"-,optional" toml:"-,optional"` + Rejournal time.Duration `hcl:"-,optional" toml:"-"` RejournalRaw string `hcl:"rejournal,optional" toml:"rejournal,optional"` // PriceLimit is the minimum gas price to enforce for acceptance into the pool @@ -189,7 +189,7 @@ type TxPoolConfig struct { GlobalQueue uint64 `hcl:"globalqueue,optional" toml:"globalqueue,optional"` // lifetime is the maximum amount of time non-executable transaction are queued - LifeTime time.Duration `hcl:"-,optional" toml:"-,optional"` + LifeTime time.Duration `hcl:"-,optional" toml:"-"` LifeTimeRaw string `hcl:"lifetime,optional" toml:"lifetime,optional"` } @@ -207,7 +207,7 @@ type SealerConfig struct { GasCeil uint64 `hcl:"gaslimit,optional" toml:"gaslimit,optional"` // GasPrice is the minimum gas price for mining a transaction - GasPrice *big.Int `hcl:"-,optional" toml:"-,optional"` + GasPrice *big.Int `hcl:"-,optional" toml:"-"` GasPriceRaw string `hcl:"gasprice,optional" toml:"gasprice,optional"` } @@ -270,11 +270,11 @@ type GpoConfig struct { Percentile uint64 `hcl:"percentile,optional" toml:"percentile,optional"` // MaxPrice is an upper bound gas price - MaxPrice *big.Int `hcl:"-,optional" toml:"-,optional"` + MaxPrice *big.Int `hcl:"-,optional" toml:"-"` MaxPriceRaw string `hcl:"maxprice,optional" toml:"maxprice,optional"` // IgnorePrice is a lower bound gas price - IgnorePrice *big.Int `hcl:"-,optional" toml:"-,optional"` + IgnorePrice *big.Int `hcl:"-,optional" toml:"-"` IgnorePriceRaw string `hcl:"ignoreprice,optional" toml:"ignoreprice,optional"` } @@ -347,7 +347,7 @@ type CacheConfig struct { Journal string `hcl:"journal,optional" toml:"journal,optional"` // Rejournal is the time interval to regenerate the journal for clean cache - Rejournal time.Duration `hcl:"-,optional" toml:"-,optional"` + Rejournal time.Duration `hcl:"-,optional" toml:"-"` RejournalRaw string `hcl:"rejournal,optional" toml:"rejournal,optional"` // NoPrefetch is used to disable prefetch of tries diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index a8127915a1..b28d7ae536 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/params" ) func TestConfigLegacy(t *testing.T) { @@ -71,8 +72,8 @@ func TestConfigLegacy(t *testing.T) { Gpo: &GpoConfig{ Blocks: 20, Percentile: 60, - MaxPrice: big.NewInt(100), - IgnorePrice: big.NewInt(2), + MaxPrice: big.NewInt(5000 * params.GWei), + IgnorePrice: big.NewInt(4), }, JsonRPC: &JsonRPCConfig{ IPCDisable: false, @@ -129,7 +130,7 @@ func TestConfigLegacy(t *testing.T) { PercGc: 25, PercSnapshot: 10, Journal: "triecache", - Rejournal: 1 * time.Hour, + Rejournal: 1 * time.Second, NoPrefetch: false, Preimages: false, TxLookupLimit: 2350000, diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index 81fc4c4630..ecc313b5b5 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -8,7 +8,6 @@ maxpeers = 30 [txpool] locals = [] -rejournal = "1h0m0s" lifetime = "1s" [miner] @@ -17,9 +16,8 @@ gaslimit = 20000000 gasprice = "30000000000" [gpo] -maxprice = "100" -ignoreprice = "2" +ignoreprice = "4" [cache] cache = 1024 -rejournal = "1h0m0s" +rejournal = "1s" From 54719fb180ad54e6fffa15afb8b3a3d7411c84e1 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 27 Jul 2022 22:27:26 +0530 Subject: [PATCH 14/70] Makefile: copy bor binary to go bin (#469) --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 46ecdec886..a8e14cf06a 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,8 @@ GOTEST = GODEBUG=cgocheck=0 go test $(GO_FLAGS) -p 1 bor: mkdir -p $(GOPATH)/bin/ go build -o $(GOBIN)/bor ./cmd/cli/main.go + cp $(GOBIN)/bor $(GOPATH)/bin/ + @echo "Done building." protoc: protoc --go_out=. --go-grpc_out=. ./internal/cli/server/proto/*.proto From 75f4411102bf5e10f5ef70184776646f16267b67 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Fri, 26 Aug 2022 17:59:05 +0530 Subject: [PATCH 15/70] fix: rename requiredblocks flag usage (#505) --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 28 ++++++++++++++-------------- eth/backend.go | 20 ++++++++++---------- eth/ethconfig/config.go | 4 ++-- eth/ethconfig/gen_config.go | 10 +++++----- eth/handler.go | 26 +++++++++++++------------- internal/cli/server/config.go | 4 ++-- 8 files changed, 48 insertions(+), 48 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 600e929706..af565d71ae 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -109,7 +109,7 @@ var ( utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, utils.LightNoSyncServeFlag, - utils.EthPeerRequiredBlocksFlag, + utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, utils.BloomFilterSizeFlag, utils.CacheFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index af5175cd63..28f8f84533 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -56,7 +56,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.EthStatsURLFlag, utils.IdentityFlag, utils.LightKDFFlag, - utils.EthPeerRequiredBlocksFlag, + utils.EthRequiredBlocksFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 79c73903fb..85f28cc942 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -249,13 +249,13 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } - EthPeerRequiredBlocksFlag = cli.StringFlag{ + EthRequiredBlocksFlag = cli.StringFlag{ Name: "eth.requiredblocks", Usage: "Comma separated block number-to-hash mappings to require for peering (=)", } LegacyWhitelistFlag = cli.StringFlag{ Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --peer.requiredblocks)", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", } BloomFilterSizeFlag = cli.Uint64Flag{ Name: "bloomfilter.size", @@ -1498,33 +1498,33 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { } } -func setPeerRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { - peerRequiredBlocks := ctx.GlobalString(EthPeerRequiredBlocksFlag.Name) +func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { + requiredBlocks := ctx.GlobalString(EthRequiredBlocksFlag.Name) - if peerRequiredBlocks == "" { + if requiredBlocks == "" { if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) { - log.Warn("The flag --rpc is deprecated and will be removed, please use --peer.requiredblocks") - peerRequiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) + log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks") + requiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) } else { return } } - cfg.PeerRequiredBlocks = make(map[uint64]common.Hash) - for _, entry := range strings.Split(peerRequiredBlocks, ",") { + cfg.RequiredBlocks = make(map[uint64]common.Hash) + for _, entry := range strings.Split(requiredBlocks, ",") { parts := strings.Split(entry, "=") if len(parts) != 2 { - Fatalf("Invalid peer required block entry: %s", entry) + Fatalf("Invalid required block entry: %s", entry) } number, err := strconv.ParseUint(parts[0], 0, 64) if err != nil { - Fatalf("Invalid peer required block number %s: %v", parts[0], err) + Fatalf("Invalid required block number %s: %v", parts[0], err) } var hash common.Hash if err = hash.UnmarshalText([]byte(parts[1])); err != nil { - Fatalf("Invalid peer required block hash %s: %v", parts[1], err) + Fatalf("Invalid required block hash %s: %v", parts[1], err) } - cfg.PeerRequiredBlocks[number] = hash + cfg.RequiredBlocks[number] = hash } } @@ -1591,7 +1591,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) setMiner(ctx, &cfg.Miner) - setPeerRequiredBlocks(ctx, cfg) + setRequiredBlocks(ctx, cfg) setLes(ctx, cfg) if ctx.GlobalIsSet(BorLogsFlag.Name) { diff --git a/eth/backend.go b/eth/backend.go index 05d6c5c927..2f31b7360f 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -243,16 +243,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { checkpoint = params.TrustedCheckpoints[genesisHash] } if eth.handler, err = newHandler(&handlerConfig{ - Database: chainDb, - Chain: eth.blockchain, - TxPool: eth.txPool, - Merger: merger, - Network: config.NetworkId, - Sync: config.SyncMode, - BloomCache: uint64(cacheLimit), - EventMux: eth.eventMux, - Checkpoint: checkpoint, - PeerRequiredBlocks: config.PeerRequiredBlocks, + Database: chainDb, + Chain: eth.blockchain, + TxPool: eth.txPool, + Merger: merger, + Network: config.NetworkId, + Sync: config.SyncMode, + BloomCache: uint64(cacheLimit), + EventMux: eth.eventMux, + Checkpoint: checkpoint, + RequiredBlocks: config.RequiredBlocks, }); err != nil { return nil, err } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 6ab43891f7..d25ae20dcb 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -140,10 +140,10 @@ type Config struct { TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. - // PeerRequiredBlocks is a set of block number -> hash mappings which must be in the + // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the // presence of these blocks for every new peer connection. - PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` // Light client options LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 874e30dffd..0930348c21 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -26,7 +26,7 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` - PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` LightEgress int `toml:",omitempty"` @@ -71,7 +71,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit - enc.PeerRequiredBlocks = c.PeerRequiredBlocks + enc.RequiredBlocks = c.RequiredBlocks enc.LightServ = c.LightServ enc.LightIngress = c.LightIngress enc.LightEgress = c.LightEgress @@ -120,7 +120,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` - PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` LightEgress *int `toml:",omitempty"` @@ -184,8 +184,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } - if dec.PeerRequiredBlocks != nil { - c.PeerRequiredBlocks = dec.PeerRequiredBlocks + if dec.RequiredBlocks != nil { + c.RequiredBlocks = dec.RequiredBlocks } if dec.LightServ != nil { c.LightServ = *dec.LightServ diff --git a/eth/handler.go b/eth/handler.go index bd31e0f117..55d3c5f8f5 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -87,7 +87,7 @@ type handlerConfig struct { EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` Checkpoint *params.TrustedCheckpoint // Hard coded checkpoint for sync challenges - PeerRequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges + RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges } type handler struct { @@ -116,7 +116,7 @@ type handler struct { txsSub event.Subscription minedBlockSub *event.TypeMuxSubscription - peerRequiredBlocks map[uint64]common.Hash + requiredBlocks map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop quitSync chan struct{} @@ -133,16 +133,16 @@ func newHandler(config *handlerConfig) (*handler, error) { config.EventMux = new(event.TypeMux) // Nicety initialization for tests } h := &handler{ - networkID: config.Network, - forkFilter: forkid.NewFilter(config.Chain), - eventMux: config.EventMux, - database: config.Database, - txpool: config.TxPool, - chain: config.Chain, - peers: newPeerSet(), - merger: config.Merger, - peerRequiredBlocks: config.PeerRequiredBlocks, - quitSync: make(chan struct{}), + networkID: config.Network, + forkFilter: forkid.NewFilter(config.Chain), + eventMux: config.EventMux, + database: config.Database, + txpool: config.TxPool, + chain: config.Chain, + peers: newPeerSet(), + merger: config.Merger, + requiredBlocks: config.RequiredBlocks, + quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the snap @@ -432,7 +432,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { }() } // If we have any explicit peer required block hashes, request them - for number, hash := range h.peerRequiredBlocks { + for number, hash := range h.requiredBlocks { resCh := make(chan *eth.Response) if _, err := peer.RequestHeadersByNumber(number, 1, 0, false, resCh); err != nil { return err diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 06ec8c16af..72f50dcdd7 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -723,7 +723,7 @@ func (c *Config) buildEth(stack *node.Node) (*ethconfig.Config, error) { // whitelist { - n.PeerRequiredBlocks = map[uint64]common.Hash{} + n.RequiredBlocks = map[uint64]common.Hash{} for k, v := range c.Whitelist { number, err := strconv.ParseUint(k, 0, 64) if err != nil { @@ -733,7 +733,7 @@ func (c *Config) buildEth(stack *node.Node) (*ethconfig.Config, error) { if err = hash.UnmarshalText([]byte(v)); err != nil { return nil, fmt.Errorf("invalid whitelist hash %s: %v", v, err) } - n.PeerRequiredBlocks[number] = hash + n.RequiredBlocks[number] = hash } } From d73d6df32bd98a4bc71b81fe92f4ec546fc3b3ff Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 30 Aug 2022 16:16:26 +0530 Subject: [PATCH 16/70] params: update meta version --- params/version.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/params/version.go b/params/version.go index ec3923a884..3984f99233 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 2 // Minor version component of the current release - VersionPatch = 17 // Patch version component of the current release - VersionMeta = "beta" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 2 // Minor version component of the current release + VersionPatch = 17 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. @@ -41,9 +41,9 @@ var VersionWithMeta = func() string { return v }() -// ArchiveVersion holds the textual version string used for Geth archives. -// e.g. "1.8.11-dea1ce05" for stable releases, or -// "1.8.13-unstable-21c059b6" for unstable releases +// ArchiveVersion holds the textual version string used for Geth archives. e.g. +// "1.8.11-dea1ce05" for stable releases, or "1.8.13-unstable-21c059b6" for unstable +// releases. func ArchiveVersion(gitCommit string) string { vsn := Version if VersionMeta != "stable" { From 7bb7bacd5042d41bb2c3a5c13e4fcc4f543afce2 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 1 Sep 2022 11:07:49 +0530 Subject: [PATCH 17/70] Merge pull request #503 from maticnetwork/shivam/sealer-gasPrice (#506) internal/cli/server: update default cli values Co-authored-by: SHIVAM SHARMA --- internal/cli/server/config.go | 20 ++++++++-------- internal/cli/server/config_legacy_test.go | 28 ++++++++++++----------- internal/cli/server/testdata/test.toml | 4 ++-- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 255cf633eb..8ee023db76 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -419,7 +419,7 @@ func DefaultConfig() *Config { TxPool: &TxPoolConfig{ Locals: []string{}, NoLocals: false, - Journal: "", + Journal: "transactions.rlp", Rejournal: 1 * time.Hour, PriceLimit: 30000000000, PriceBump: 10, @@ -432,8 +432,8 @@ func DefaultConfig() *Config { Sealer: &SealerConfig{ Enabled: false, Etherbase: "", - GasCeil: 20000000, - GasPrice: big.NewInt(30 * params.GWei), + GasCeil: 30_000_000, + GasPrice: big.NewInt(1 * params.GWei), ExtraData: "", }, Gpo: &GpoConfig{ @@ -453,22 +453,22 @@ func DefaultConfig() *Config { Prefix: "", Host: "localhost", API: []string{"eth", "net", "web3", "txpool", "bor"}, - Cors: []string{"*"}, - VHost: []string{"*"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, Ws: &APIConfig{ Enabled: false, Port: 8546, Prefix: "", Host: "localhost", - API: []string{"web3", "net"}, - Cors: []string{"*"}, - VHost: []string{"*"}, + API: []string{"net", "web3"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, Graphql: &APIConfig{ Enabled: false, - Cors: []string{"*"}, - VHost: []string{"*"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, }, Ethstats: "", diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index b28d7ae536..b11e9646f4 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -14,10 +14,10 @@ import ( func TestConfigLegacy(t *testing.T) { readFile := func(path string) { - config, err := readLegacyConfig(path) + expectedConfig, err := readLegacyConfig(path) assert.NoError(t, err) - assert.Equal(t, config, &Config{ + testConfig := &Config{ Chain: "mainnet", Identity: Hostname(), RequiredBlocks: map[string]string{ @@ -52,7 +52,7 @@ func TestConfigLegacy(t *testing.T) { TxPool: &TxPoolConfig{ Locals: []string{}, NoLocals: false, - Journal: "", + Journal: "transactions.rlp", Rejournal: 1 * time.Hour, PriceLimit: 30000000000, PriceBump: 10, @@ -65,8 +65,8 @@ func TestConfigLegacy(t *testing.T) { Sealer: &SealerConfig{ Enabled: false, Etherbase: "", - GasCeil: 20000000, - GasPrice: big.NewInt(30000000000), + GasCeil: 30000000, + GasPrice: big.NewInt(1 * params.GWei), ExtraData: "", }, Gpo: &GpoConfig{ @@ -86,22 +86,22 @@ func TestConfigLegacy(t *testing.T) { Prefix: "", Host: "localhost", API: []string{"eth", "net", "web3", "txpool", "bor"}, - Cors: []string{"*"}, - VHost: []string{"*"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, Ws: &APIConfig{ Enabled: false, Port: 8546, Prefix: "", Host: "localhost", - API: []string{"web3", "net"}, - Cors: []string{"*"}, - VHost: []string{"*"}, + API: []string{"net", "web3"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, Graphql: &APIConfig{ Enabled: false, - Cors: []string{"*"}, - VHost: []string{"*"}, + Cors: []string{"localhost"}, + VHost: []string{"localhost"}, }, }, Ethstats: "", @@ -149,7 +149,9 @@ func TestConfigLegacy(t *testing.T) { Enabled: false, Period: 0, }, - }) + } + + assert.Equal(t, expectedConfig, testConfig) } // read file in hcl format diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index ecc313b5b5..2277e03664 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -12,8 +12,8 @@ lifetime = "1s" [miner] mine = false -gaslimit = 20000000 -gasprice = "30000000000" +gaslimit = 30000000 +gasprice = "1000000000" [gpo] ignoreprice = "4" From d2eb2df62e9571d958cf86c8e6aa4621afb7e1eb Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Wed, 7 Sep 2022 13:35:22 +0400 Subject: [PATCH 18/70] changed 'requiredblocks' flag back to 'eth.requiredblocks' --- builder/files/config.toml | 2 +- docs/config.md | 2 ++ internal/cli/server/config.go | 2 +- internal/cli/server/flags.go | 2 +- internal/cli/server/testdata/test.toml | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index d8503e4351..84c3327bbf 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -12,7 +12,7 @@ syncmode = "full" # snapshot = true # ethstats = "" -# [requiredblocks] +# ["eth.requiredblocks"] [p2p] # maxpeers = 1 diff --git a/docs/config.md b/docs/config.md index 196a0bf388..aebd9e12b9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -19,6 +19,8 @@ gcmode = "full" snapshot = true ethstats = "" +["eth.requiredblocks"] + [p2p] maxpeers = 30 maxpendpeers = 50 diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 8ee023db76..f91e336ace 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -45,7 +45,7 @@ type Config struct { Identity string `hcl:"identity,optional" toml:"identity,optional"` // RequiredBlocks is a list of required (block number, hash) pairs to accept - RequiredBlocks map[string]string `hcl:"requiredblocks,optional" toml:"requiredblocks,optional"` + RequiredBlocks map[string]string `hcl:"eth.requiredblocks,optional" toml:"eth.requiredblocks,optional"` // LogLevel is the level of the logs to put out LogLevel string `hcl:"log-level,optional" toml:"log-level,optional"` diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index bbc7f44036..e19e578c57 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -56,7 +56,7 @@ func (c *Command) Flags() *flagset.Flagset { Default: c.cliConfig.GcMode, }) f.MapStringFlag(&flagset.MapStringFlag{ - Name: "requiredblocks", + Name: "eth.requiredblocks", Usage: "Comma separated block number-to-hash mappings to enforce (=)", Value: &c.cliConfig.RequiredBlocks, }) diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index 2277e03664..6d07abdad7 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -1,6 +1,6 @@ datadir = "./data" -[requiredblocks] +["eth.requiredblocks"] a = "b" [p2p] From 7c6f0a67e6e01430ea9304a7adef80ede21e903f Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Wed, 7 Sep 2022 16:21:55 +0400 Subject: [PATCH 19/70] updated tests, and removed requiredblocks from json and hcl --- internal/cli/server/command_test.go | 2 +- internal/cli/server/config_test.go | 3 --- internal/cli/server/testdata/test.hcl | 4 ---- internal/cli/server/testdata/test.json | 3 --- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/internal/cli/server/command_test.go b/internal/cli/server/command_test.go index 9006559d0f..ab28de5ee6 100644 --- a/internal/cli/server/command_test.go +++ b/internal/cli/server/command_test.go @@ -24,7 +24,7 @@ func TestFlags(t *testing.T) { "--dev.period", "2", "--datadir", "./data", "--maxpeers", "30", - "--requiredblocks", "a=b", + "--eth.requiredblocks", "a=b", "--http.api", "eth,web3,bor", } err := c.extractFlags(args) diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index bf7787ac2e..5f3118996b 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -108,9 +108,6 @@ func TestConfigLoadFile(t *testing.T) { assert.Equal(t, config, &Config{ DataDir: "./data", - RequiredBlocks: map[string]string{ - "a": "b", - }, P2P: &P2PConfig{ MaxPeers: 30, }, diff --git a/internal/cli/server/testdata/test.hcl b/internal/cli/server/testdata/test.hcl index 208fdc51c1..44138970fc 100644 --- a/internal/cli/server/testdata/test.hcl +++ b/internal/cli/server/testdata/test.hcl @@ -1,9 +1,5 @@ datadir = "./data" -requiredblocks = { - a = "b" -} - p2p { maxpeers = 30 } diff --git a/internal/cli/server/testdata/test.json b/internal/cli/server/testdata/test.json index 467835e755..a08e5aceb1 100644 --- a/internal/cli/server/testdata/test.json +++ b/internal/cli/server/testdata/test.json @@ -1,8 +1,5 @@ { "datadir": "./data", - "requiredblocks": { - "a": "b" - }, "p2p": { "maxpeers": 30 }, From 2cf5945321290da53edebf71dd7e4a45b230d388 Mon Sep 17 00:00:00 2001 From: Jerry Date: Mon, 5 Sep 2022 23:25:46 -0700 Subject: [PATCH 20/70] Remove orphaned containers when shutdown devnet This will potentially resolve "ERROR: network docker_default has active endpoints" problem --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 336b2fea56..639a68703f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -168,7 +168,7 @@ jobs: if: always() run: | cd matic-cli/devnet - docker compose down + docker compose down --remove-orphans cd - mkdir -p ${{ github.run_id }}/matic-cli sudo mv bor ${{ github.run_id }} From c91d4ca1fa4fd0b6555ce8f3734e91a1d5761815 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 22 Sep 2022 10:36:21 +0530 Subject: [PATCH 21/70] rpc, ethclient: cater 'finalized' and 'safe' blocks in ethclient (#517) * rpc: add finalized and safe block types * ethclient: honour safe and finalized block requests * fix: remove BlockByNumberWithoutTx function --- ethclient/ethclient.go | 8 ++++++++ ethclient/gethclient/gethclient.go | 8 ++++++++ rpc/types.go | 8 +++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 6e8915358b..c0b91b8569 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -537,6 +537,14 @@ func toBlockNumArg(number *big.Int) string { if number.Cmp(pending) == 0 { return "pending" } + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { + return "finalized" + } + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { + return "safe" + } return hexutil.EncodeBig(number) } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index 538e23727d..ae147a40a3 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -187,6 +187,14 @@ func toBlockNumArg(number *big.Int) string { if number.Cmp(pending) == 0 { return "pending" } + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { + return "finalized" + } + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { + return "safe" + } return hexutil.EncodeBig(number) } diff --git a/rpc/types.go b/rpc/types.go index 46b08caf68..e4d40ab98f 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -61,9 +61,11 @@ type jsonWriter interface { type BlockNumber int64 const ( - PendingBlockNumber = BlockNumber(-2) - LatestBlockNumber = BlockNumber(-1) - EarliestBlockNumber = BlockNumber(0) + SafeBlockNumber = BlockNumber(-4) + FinalizedBlockNumber = BlockNumber(-3) + PendingBlockNumber = BlockNumber(-2) + LatestBlockNumber = BlockNumber(-1) + EarliestBlockNumber = BlockNumber(0) ) // UnmarshalJSON parses the given JSON fragment into a BlockNumber. It supports: From ebbe100ff16d378a5b4860561ebc03f5d6423e98 Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 27 Sep 2022 22:54:17 -0700 Subject: [PATCH 22/70] Change heimdall branch to develop in CI --- .github/matic-cli-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/matic-cli-config.yml b/.github/matic-cli-config.yml index f643f4b192..c7dab39b0e 100644 --- a/.github/matic-cli-config.yml +++ b/.github/matic-cli-config.yml @@ -8,4 +8,4 @@ numOfNonValidators: 0 ethURL: http://ganache:9545 devnetType: docker borDockerBuildContext: "../../bor" -heimdallDockerBuildContext: "https://github.com/maticnetwork/heimdall.git#v0.3.0-dev" \ No newline at end of file +heimdallDockerBuildContext: "https://github.com/maticnetwork/heimdall.git#develop" \ No newline at end of file From 43e5d3aa090c8f88b3ced29ff8bb1bbb16fa0dca Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Thu, 29 Sep 2022 14:25:27 +0530 Subject: [PATCH 23/70] PR#518 and PR#529 -> qa (#532) * Added script to generate config.toml fromstart.sh (#518) * added go and bash script to get config out of start.sh and updated flagset.go * changed 'requiredblocks' flag back to 'eth.requiredblocks' * updated script * changed 'requiredblocks' flag back to 'eth.requiredblocks' * updated tests, and removed requiredblocks from json and hcl * addressed comments * internal/cli/server: fix flag behaviour (#529) * remove setting maxpeers to 0 for nodiscover flag * set default prometheus and open-collector endpoint * skip building grpc address from pprof address and port * fix: linters * fix and improve tests * use loopback address for prometheus and open-collector endpoint * add logs for prometheus and open-collector setup * updated the script to handle prometheus-addr * updated builder/files/config.toml Co-authored-by: Pratik Patil Co-authored-by: Manav Darji --- builder/files/config.toml | 4 +- go.mod | 1 + go.sum | 2 + internal/cli/flagset/flagset.go | 9 + internal/cli/server/config.go | 7 +- internal/cli/server/config_legacy_test.go | 151 +---- internal/cli/server/flags.go | 2 +- internal/cli/server/server.go | 4 + internal/cli/server/testdata/test.toml | 6 +- scripts/getconfig.go | 687 ++++++++++++++++++++++ scripts/getconfig.sh | 87 +++ 11 files changed, 816 insertions(+), 144 deletions(-) create mode 100644 scripts/getconfig.go create mode 100755 scripts/getconfig.sh diff --git a/builder/files/config.toml b/builder/files/config.toml index 84c3327bbf..ce9fd782d0 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -96,8 +96,8 @@ syncmode = "full" [telemetry] metrics = true # expensive = false - prometheus-addr = "127.0.0.1:7071" - # opencollector-endpoint = "" + # prometheus-addr = "127.0.0.1:7071" + # opencollector-endpoint = "127.0.0.1:4317" # [telemetry.influx] # influxdb = false # endpoint = "" diff --git a/go.mod b/go.mod index 39bbeb240d..e31612cfe3 100644 --- a/go.mod +++ b/go.mod @@ -119,6 +119,7 @@ require ( github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.1.1 // indirect diff --git a/go.sum b/go.sum index 88c9152eef..593c8e11d0 100644 --- a/go.sum +++ b/go.sum @@ -391,6 +391,8 @@ github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mo github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= diff --git a/internal/cli/flagset/flagset.go b/internal/cli/flagset/flagset.go index d9e204f0e4..933fe59060 100644 --- a/internal/cli/flagset/flagset.go +++ b/internal/cli/flagset/flagset.go @@ -44,6 +44,15 @@ func (f *Flagset) Help() string { return str + strings.Join(items, "\n\n") } +func (f *Flagset) GetAllFlags() []string { + flags := []string{} + for _, flag := range f.flags { + flags = append(flags, flag.Name) + } + + return flags +} + // MarkDown implements cli.MarkDown interface func (f *Flagset) MarkDown() string { if len(f.flags) == 0 { diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index f91e336ace..a4f642a6e3 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -475,8 +475,8 @@ func DefaultConfig() *Config { Telemetry: &TelemetryConfig{ Enabled: false, Expensive: false, - PrometheusAddr: "", - OpenCollectorEndpoint: "", + PrometheusAddr: "127.0.0.1:7071", + OpenCollectorEndpoint: "127.0.0.1:4317", InfluxDB: &InfluxDBConfig{ V1Enabled: false, Endpoint: "", @@ -991,8 +991,7 @@ func (c *Config) buildNode() (*node.Config, error) { } if c.P2P.NoDiscover { - // Disable networking, for now, we will not even allow incomming connections - cfg.P2P.MaxPeers = 0 + // Disable peer discovery cfg.P2P.NoDiscovery = true } diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index b11e9646f4..29cefdd7bf 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -6,9 +6,6 @@ import ( "time" "github.com/stretchr/testify/assert" - - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/params" ) func TestConfigLegacy(t *testing.T) { @@ -17,139 +14,23 @@ func TestConfigLegacy(t *testing.T) { expectedConfig, err := readLegacyConfig(path) assert.NoError(t, err) - testConfig := &Config{ - Chain: "mainnet", - Identity: Hostname(), - RequiredBlocks: map[string]string{ - "a": "b", - }, - LogLevel: "INFO", - DataDir: "./data", - P2P: &P2PConfig{ - MaxPeers: 30, - MaxPendPeers: 50, - Bind: "0.0.0.0", - Port: 30303, - NoDiscover: false, - NAT: "any", - Discovery: &P2PDiscovery{ - V5Enabled: false, - Bootnodes: []string{}, - BootnodesV4: []string{}, - BootnodesV5: []string{}, - StaticNodes: []string{}, - TrustedNodes: []string{}, - DNS: []string{}, - }, - }, - Heimdall: &HeimdallConfig{ - URL: "http://localhost:1317", - Without: false, - }, - SyncMode: "full", - GcMode: "full", - Snapshot: true, - TxPool: &TxPoolConfig{ - Locals: []string{}, - NoLocals: false, - Journal: "transactions.rlp", - Rejournal: 1 * time.Hour, - PriceLimit: 30000000000, - PriceBump: 10, - AccountSlots: 16, - GlobalSlots: 32768, - AccountQueue: 16, - GlobalQueue: 32768, - LifeTime: 1 * time.Second, - }, - Sealer: &SealerConfig{ - Enabled: false, - Etherbase: "", - GasCeil: 30000000, - GasPrice: big.NewInt(1 * params.GWei), - ExtraData: "", - }, - Gpo: &GpoConfig{ - Blocks: 20, - Percentile: 60, - MaxPrice: big.NewInt(5000 * params.GWei), - IgnorePrice: big.NewInt(4), - }, - JsonRPC: &JsonRPCConfig{ - IPCDisable: false, - IPCPath: "", - GasCap: ethconfig.Defaults.RPCGasCap, - TxFeeCap: ethconfig.Defaults.RPCTxFeeCap, - Http: &APIConfig{ - Enabled: false, - Port: 8545, - Prefix: "", - Host: "localhost", - API: []string{"eth", "net", "web3", "txpool", "bor"}, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - Ws: &APIConfig{ - Enabled: false, - Port: 8546, - Prefix: "", - Host: "localhost", - API: []string{"net", "web3"}, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - Graphql: &APIConfig{ - Enabled: false, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - }, - Ethstats: "", - Telemetry: &TelemetryConfig{ - Enabled: false, - Expensive: false, - PrometheusAddr: "", - OpenCollectorEndpoint: "", - InfluxDB: &InfluxDBConfig{ - V1Enabled: false, - Endpoint: "", - Database: "", - Username: "", - Password: "", - Tags: map[string]string{}, - V2Enabled: false, - Token: "", - Bucket: "", - Organization: "", - }, - }, - Cache: &CacheConfig{ - Cache: 1024, - PercDatabase: 50, - PercTrie: 15, - PercGc: 25, - PercSnapshot: 10, - Journal: "triecache", - Rejournal: 1 * time.Second, - NoPrefetch: false, - Preimages: false, - TxLookupLimit: 2350000, - }, - Accounts: &AccountsConfig{ - Unlock: []string{}, - PasswordFile: "", - AllowInsecureUnlock: false, - UseLightweightKDF: false, - DisableBorWallet: true, - }, - GRPC: &GRPCConfig{ - Addr: ":3131", - }, - Developer: &DeveloperConfig{ - Enabled: false, - Period: 0, - }, + testConfig := DefaultConfig() + + testConfig.DataDir = "./data" + testConfig.Snapshot = false + testConfig.RequiredBlocks = map[string]string{ + "31000000": "0x2087b9e2b353209c2c21e370c82daa12278efd0fe5f0febe6c29035352cf050e", + "32000000": "0x875500011e5eecc0c554f95d07b31cf59df4ca2505f4dbbfffa7d4e4da917c68", } + testConfig.P2P.MaxPeers = 30 + testConfig.TxPool.Locals = []string{} + testConfig.TxPool.LifeTime = time.Second + testConfig.Sealer.Enabled = true + testConfig.Sealer.GasCeil = 30000000 + testConfig.Sealer.GasPrice = big.NewInt(1000000000) + testConfig.Gpo.IgnorePrice = big.NewInt(4) + testConfig.Cache.Cache = 1024 + testConfig.Cache.Rejournal = time.Second assert.Equal(t, expectedConfig, testConfig) } diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index e19e578c57..835ff0ab95 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -57,7 +57,7 @@ func (c *Command) Flags() *flagset.Flagset { }) f.MapStringFlag(&flagset.MapStringFlag{ Name: "eth.requiredblocks", - Usage: "Comma separated block number-to-hash mappings to enforce (=)", + Usage: "Comma separated block number-to-hash mappings to require for peering (=)", Value: &c.cliConfig.RequiredBlocks, }) f.BoolFlag(&flagset.BoolFlag{ diff --git a/internal/cli/server/server.go b/internal/cli/server/server.go index cd706c1a9d..adacb44ce7 100644 --- a/internal/cli/server/server.go +++ b/internal/cli/server/server.go @@ -281,6 +281,8 @@ func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error } }() + log.Info("Enabling metrics export to prometheus", "path", fmt.Sprintf("http://%s/debug/metrics/prometheus", config.PrometheusAddr)) + } if config.OpenCollectorEndpoint != "" { @@ -322,6 +324,8 @@ func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error // set the tracer s.tracer = tracerProvider + + log.Info("Open collector tracing started", "address", config.OpenCollectorEndpoint) } return nil diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index 6d07abdad7..4ccc644ee9 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -1,7 +1,9 @@ datadir = "./data" +snapshot = false ["eth.requiredblocks"] -a = "b" +"31000000" = "0x2087b9e2b353209c2c21e370c82daa12278efd0fe5f0febe6c29035352cf050e" +"32000000" = "0x875500011e5eecc0c554f95d07b31cf59df4ca2505f4dbbfffa7d4e4da917c68" [p2p] maxpeers = 30 @@ -11,7 +13,7 @@ locals = [] lifetime = "1s" [miner] -mine = false +mine = true gaslimit = 30000000 gasprice = "1000000000" diff --git a/scripts/getconfig.go b/scripts/getconfig.go new file mode 100644 index 0000000000..689ed68fbf --- /dev/null +++ b/scripts/getconfig.go @@ -0,0 +1,687 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "strconv" + "strings" + + "github.com/pelletier/go-toml" + + "github.com/ethereum/go-ethereum/internal/cli/server" +) + +// YesFV: Both, Flags and their values has changed +// YesF: Only the Flag has changed, not their value +var flagMap = map[string][]string{ + "networkid": {"notABoolFlag", "YesFV"}, + "miner.gastarget": {"notABoolFlag", "No"}, + "pprof": {"BoolFlag", "No"}, + "pprof.port": {"notABoolFlag", "No"}, + "pprof.addr": {"notABoolFlag", "No"}, + "pprof.memprofilerate": {"notABoolFlag", "No"}, + "pprof.blockprofilerate": {"notABoolFlag", "No"}, + "pprof.cpuprofile": {"notABoolFlag", "No"}, + "jsonrpc.corsdomain": {"notABoolFlag", "YesF"}, + "jsonrpc.vhosts": {"notABoolFlag", "YesF"}, + "http.modules": {"notABoolFlag", "YesF"}, + "ws.modules": {"notABoolFlag", "YesF"}, + "config": {"notABoolFlag", "No"}, + "datadir.ancient": {"notABoolFlag", "No"}, + "datadir.minfreedisk": {"notABoolFlag", "No"}, + "usb": {"BoolFlag", "No"}, + "pcscdpath": {"notABoolFlag", "No"}, + "mainnet": {"BoolFlag", "No"}, + "goerli": {"BoolFlag", "No"}, + "bor-mumbai": {"BoolFlag", "No"}, + "bor-mainnet": {"BoolFlag", "No"}, + "rinkeby": {"BoolFlag", "No"}, + "ropsten": {"BoolFlag", "No"}, + "sepolia": {"BoolFlag", "No"}, + "kiln": {"BoolFlag", "No"}, + "exitwhensynced": {"BoolFlag", "No"}, + "light.serve": {"notABoolFlag", "No"}, + "light.ingress": {"notABoolFlag", "No"}, + "light.egress": {"notABoolFlag", "No"}, + "light.maxpeers": {"notABoolFlag", "No"}, + "ulc.servers": {"notABoolFlag", "No"}, + "ulc.fraction": {"notABoolFlag", "No"}, + "ulc.onlyannounce": {"BoolFlag", "No"}, + "light.nopruning": {"BoolFlag", "No"}, + "light.nosyncserve": {"BoolFlag", "No"}, + "dev.gaslimit": {"notABoolFlag", "No"}, + "ethash.cachedir": {"notABoolFlag", "No"}, + "ethash.cachesinmem": {"notABoolFlag", "No"}, + "ethash.cachesondisk": {"notABoolFlag", "No"}, + "ethash.cacheslockmmap": {"BoolFlag", "No"}, + "ethash.dagdir": {"notABoolFlag", "No"}, + "ethash.dagsinmem": {"notABoolFlag", "No"}, + "ethash.dagsondisk": {"notABoolFlag", "No"}, + "ethash.dagslockmmap": {"BoolFlag", "No"}, + "fdlimit": {"notABoolFlag", "No"}, + "signer": {"notABoolFlag", "No"}, + "authrpc.jwtsecret": {"notABoolFlag", "No"}, + "authrpc.addr": {"notABoolFlag", "No"}, + "authrpc.port": {"notABoolFlag", "No"}, + "authrpc.vhosts": {"notABoolFlag", "No"}, + "graphql.corsdomain": {"notABoolFlag", "No"}, + "graphql.vhosts": {"notABoolFlag", "No"}, + "rpc.evmtimeout": {"notABoolFlag", "No"}, + "rpc.allow-unprotected-txs": {"BoolFlag", "No"}, + "jspath": {"notABoolFlag", "No"}, + "exec": {"notABoolFlag", "No"}, + "preload": {"notABoolFlag", "No"}, + "discovery.dns": {"notABoolFlag", "No"}, + "netrestrict": {"notABoolFlag", "No"}, + "nodekey": {"notABoolFlag", "No"}, + "nodekeyhex": {"notABoolFlag", "No"}, + "miner.threads": {"notABoolFlag", "No"}, + "miner.notify": {"notABoolFlag", "No"}, + "miner.notify.full": {"BoolFlag", "No"}, + "miner.recommit": {"notABoolFlag", "No"}, + "miner.noverify": {"BoolFlag", "No"}, + "vmdebug": {"BoolFlag", "No"}, + "fakepow": {"BoolFlag", "No"}, + "nocompaction": {"BoolFlag", "No"}, + "metrics.addr": {"notABoolFlag", "No"}, + "metrics.port": {"notABoolFlag", "No"}, + "whitelist": {"notABoolFlag", "No"}, + "snapshot": {"BoolFlag", "YesF"}, + "bloomfilter.size": {"notABoolFlag", "No"}, + "bor.logs": {"BoolFlag", "No"}, + "override.arrowglacier": {"notABoolFlag", "No"}, + "override.terminaltotaldifficulty": {"notABoolFlag", "No"}, + "verbosity": {"notABoolFlag", "YesFV"}, + "ws.origins": {"notABoolFlag", "YesF"}, +} + +// map from cli flags to corresponding toml tags +var nameTagMap = map[string]string{ + "chain": "chain", + "identity": "identity", + "log-level": "log-level", + "datadir": "datadir", + "keystore": "keystore", + "syncmode": "syncmode", + "gcmode": "gcmode", + "eth.requiredblocks": "eth.requiredblocks", + "0-snapshot": "snapshot", + "url": "bor.heimdall", + "bor.without": "bor.withoutheimdall", + "grpc-address": "bor.heimdallgRPC", + "locals": "txpool.locals", + "nolocals": "txpool.nolocals", + "journal": "txpool.journal", + "rejournal": "txpool.rejournal", + "pricelimit": "txpool.pricelimit", + "pricebump": "txpool.pricebump", + "accountslots": "txpool.accountslots", + "globalslots": "txpool.globalslots", + "accountqueue": "txpool.accountqueue", + "globalqueue": "txpool.globalqueue", + "lifetime": "txpool.lifetime", + "mine": "mine", + "etherbase": "miner.etherbase", + "extradata": "miner.extradata", + "gaslimit": "miner.gaslimit", + "gasprice": "miner.gasprice", + "ethstats": "ethstats", + "blocks": "gpo.blocks", + "percentile": "gpo.percentile", + "maxprice": "gpo.maxprice", + "ignoreprice": "gpo.ignoreprice", + "cache": "cache", + "1-database": "cache.database", + "trie": "cache.trie", + "trie.journal": "cache.journal", + "trie.rejournal": "cache.rejournal", + "gc": "cache.gc", + "1-snapshot": "cache.snapshot", + "noprefetch": "cache.noprefetch", + "preimages": "cache.preimages", + "txlookuplimit": "txlookuplimit", + "gascap": "rpc.gascap", + "txfeecap": "rpc.txfeecap", + "ipcdisable": "ipcdisable", + "ipcpath": "ipcpath", + "1-corsdomain": "http.corsdomain", + "1-vhosts": "http.vhosts", + "2-corsdomain": "ws.corsdomain", + "2-vhosts": "ws.vhosts", + "3-corsdomain": "graphql.corsdomain", + "3-vhosts": "graphql.vhosts", + "1-enabled": "http", + "1-host": "http.addr", + "1-port": "http.port", + "1-prefix": "http.rpcprefix", + "1-api": "http.api", + "2-enabled": "ws", + "2-host": "ws.addr", + "2-port": "ws.port", + "2-prefix": "ws.rpcprefix", + "2-api": "ws.api", + "3-enabled": "graphql", + "bind": "bind", + "0-port": "port", + "bootnodes": "bootnodes", + "maxpeers": "maxpeers", + "maxpendpeers": "maxpendpeers", + "nat": "nat", + "nodiscover": "nodiscover", + "v5disc": "v5disc", + "metrics": "metrics", + "expensive": "metrics.expensive", + "influxdb": "metrics.influxdb", + "endpoint": "metrics.influxdb.endpoint", + "0-database": "metrics.influxdb.database", + "username": "metrics.influxdb.username", + "0-password": "metrics.influxdb.password", + "tags": "metrics.influxdb.tags", + "prometheus-addr": "metrics.prometheus-addr", + "opencollector-endpoint": "metrics.opencollector-endpoint", + "influxdbv2": "metrics.influxdbv2", + "token": "metrics.influxdb.token", + "bucket": "metrics.influxdb.bucket", + "organization": "metrics.influxdb.organization", + "unlock": "unlock", + "1-password": "password", + "allow-insecure-unlock": "allow-insecure-unlock", + "lightkdf": "lightkdf", + "disable-bor-wallet": "disable-bor-wallet", + "addr": "grpc.addr", + "dev": "dev", + "period": "dev.period", +} + +var removedFlagsAndValues = map[string]string{} + +var replacedFlagsMapFlagAndValue = map[string]map[string]map[string]string{ + "networkid": { + "flag": { + "networkid": "chain", + }, + "value": { + "'137'": "mainnet", + "137": "mainnet", + "'80001'": "mumbai", + "80001": "mumbai", + }, + }, + "verbosity": { + "flag": { + "verbosity": "log-level", + }, + "value": { + "0": "SILENT", + "1": "ERROR", + "2": "WARN", + "3": "INFO", + "4": "DEBUG", + "5": "DETAIL", + }, + }, +} + +var replacedFlagsMapFlag = map[string]string{ + "ws.origins": "ws.corsdomain", +} + +var currentBoolFlags = []string{ + "snapshot", + "bor.withoutheimdall", + "txpool.nolocals", + "mine", + "cache.noprefetch", + "cache.preimages", + "ipcdisable", + "http", + "ws", + "graphql", + "nodiscover", + "v5disc", + "metrics", + "metrics.expensive", + "metrics.influxdb", + "metrics.influxdbv2", + "allow-insecure-unlock", + "lightkdf", + "disable-bor-wallet", + "dev", +} + +func contains(s []string, str string) (bool, int) { + for ind, v := range s { + if v == str || v == "-"+str || v == "--"+str { + return true, ind + } + } + + return false, -1 +} + +func indexOf(s []string, str string) int { + for k, v := range s { + if v == str || v == "-"+str || v == "--"+str { + return k + } + } + + return -1 +} + +func remove1(s []string, idx int) []string { + removedFlagsAndValues[s[idx]] = "" + return append(s[:idx], s[idx+1:]...) +} + +func remove2(s []string, idx int) []string { + removedFlagsAndValues[s[idx]] = s[idx+1] + return append(s[:idx], s[idx+2:]...) +} + +func checkFlag(allFlags []string, checkFlags []string) []string { + outOfDateFlags := []string{} + + for _, flag := range checkFlags { + t1, _ := contains(allFlags, flag) + if !t1 { + outOfDateFlags = append(outOfDateFlags, flag) + } + } + + return outOfDateFlags +} + +func checkFileExists(path string) bool { + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + fmt.Println("WARN: File does not exist", path) + return false + } else { + return true + } +} + +func writeTempStaticJSON(path string) { + data, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + var conf interface{} + if err := json.Unmarshal(data, &conf); err != nil { + log.Fatal(err) + } + + temparr := []string{} + for _, item := range conf.([]interface{}) { + temparr = append(temparr, item.(string)) + } + + // write to a temp file + err = os.WriteFile("./tempStaticNodes.json", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } +} + +func writeTempStaticTrustedTOML(path string) { + data, err := toml.LoadFile(path) + if err != nil { + log.Fatal(err) + } + + if data.Has("Node.P2P.StaticNodes") { + temparr := []string{} + for _, item := range data.Get("Node.P2P.StaticNodes").([]interface{}) { + temparr = append(temparr, item.(string)) + } + + err = os.WriteFile("./tempStaticNodes.toml", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } + } + + if data.Has("Node.P2P.TrustedNodes") { + temparr := []string{} + for _, item := range data.Get("Node.P2P.TrustedNodes").([]interface{}) { + temparr = append(temparr, item.(string)) + } + + err = os.WriteFile("./tempTrustedNodes.toml", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } + } +} + +func getStaticTrustedNodes(args []string) { + // if config flag is passed, it should be only a .toml file + t1, t2 := contains(args, "config") + // nolint: nestif + if t1 { + path := args[t2+1] + if !checkFileExists(path) { + return + } + + if path[len(path)-4:] == "toml" { + writeTempStaticTrustedTOML(path) + } else { + fmt.Println("only TOML config file is supported through CLI") + } + } else { + path := "~/.bor/data/bor/static-nodes.json" + if !checkFileExists(path) { + return + } + writeTempStaticJSON(path) + } +} + +func getFlagsToCheck(args []string) []string { + flagsToCheck := []string{} + + for _, item := range args { + if strings.HasPrefix(item, "-") { + if item[1] == '-' { + temp := item[2:] + flagsToCheck = append(flagsToCheck, temp) + } else { + temp := item[1:] + flagsToCheck = append(flagsToCheck, temp) + } + } + } + + return flagsToCheck +} + +func getFlagType(flag string) string { + return flagMap[flag][0] +} + +func updateArgsClean(args []string, outOfDateFlags []string) []string { + updatedArgs := []string{} + updatedArgs = append(updatedArgs, args...) + + // iterate through outOfDateFlags and remove the flags from updatedArgs along with their value (if any) + for _, item := range outOfDateFlags { + idx := indexOf(updatedArgs, item) + + if getFlagType(item) == "BoolFlag" { + // remove the element at index idx + updatedArgs = remove1(updatedArgs, idx) + } else { + // remove the element at index idx and idx + 1 + updatedArgs = remove2(updatedArgs, idx) + } + } + + return updatedArgs +} + +func updateArgsAdd(args []string) []string { + for flag, value := range removedFlagsAndValues { + if strings.HasPrefix(flag, "--") { + flag = flag[2:] + } else { + flag = flag[1:] + } + + if flagMap[flag][1] == "YesFV" { + temp := "--" + replacedFlagsMapFlagAndValue[flag]["flag"][flag] + " " + replacedFlagsMapFlagAndValue[flag]["value"][value] + args = append(args, temp) + } else if flagMap[flag][1] == "YesF" { + temp := "--" + replacedFlagsMapFlag[flag] + " " + value + args = append(args, temp) + } + } + + return args +} + +func handlePrometheus(args []string, updatedArgs []string) []string { + var newUpdatedArgs []string + + mAddr := "" + mPort := "" + + pAddr := "" + pPort := "" + + newUpdatedArgs = append(newUpdatedArgs, updatedArgs...) + + for i, val := range args { + if strings.Contains(val, "metrics.addr") && strings.HasPrefix(val, "-") { + mAddr = args[i+1] + } + + if strings.Contains(val, "metrics.port") && strings.HasPrefix(val, "-") { + mPort = args[i+1] + } + + if strings.Contains(val, "pprof.addr") && strings.HasPrefix(val, "-") { + pAddr = args[i+1] + } + + if strings.Contains(val, "pprof.port") && strings.HasPrefix(val, "-") { + pPort = args[i+1] + } + } + + if mAddr != "" && mPort != "" { + newUpdatedArgs = append(newUpdatedArgs, "--metrics.prometheus-addr") + newUpdatedArgs = append(newUpdatedArgs, mAddr+":"+mPort) + } else if pAddr != "" && pPort != "" { + newUpdatedArgs = append(newUpdatedArgs, "--metrics.prometheus-addr") + newUpdatedArgs = append(newUpdatedArgs, pAddr+":"+pPort) + } + + return newUpdatedArgs +} + +func dumpFlags(args []string) { + err := os.WriteFile("./temp", []byte(strings.Join(args, " ")), 0600) + if err != nil { + fmt.Println("Error in WriteFile") + } else { + fmt.Println("WriteFile Done") + } +} + +// nolint: gocognit +func commentFlags(path string, updatedArgs []string) { + const cache = "[cache]" + + const telemetry = "[telemetry]" + + // snapshot: "[cache]" + cacheFlag := 0 + + // corsdomain, vhosts, enabled, host, api, port, prefix: "[p2p]", " [jsonrpc.http]", " [jsonrpc.ws]", " [jsonrpc.graphql]" + p2pHttpWsGraphFlag := -1 + + // database: "[telemetry]", "[cache]" + databaseFlag := -1 + + // password: "[telemetry]", "[accounts]" + passwordFlag := -1 + + ignoreLineFlag := false + + input, err := os.ReadFile(path) + if err != nil { + log.Fatalln(err) + } + + lines := strings.Split(string(input), "\n") + + var newLines []string + newLines = append(newLines, lines...) + + for i, line := range lines { + if line == cache { + cacheFlag += 1 + } + + if line == "[p2p]" || line == " [jsonrpc.http]" || line == " [jsonrpc.ws]" || line == " [jsonrpc.graphql]" { + p2pHttpWsGraphFlag += 1 + } + + if line == telemetry || line == cache { + databaseFlag += 1 + } + + if line == telemetry || line == "[accounts]" { + passwordFlag += 1 + } + + if line == "[\"eth.requiredblocks\"]" || line == " [telemetry.influx.tags]" { + ignoreLineFlag = true + } else if line != "" { + if strings.HasPrefix(strings.Fields(line)[0], "[") { + ignoreLineFlag = false + } + } + + // nolint: nestif + if !(strings.HasPrefix(line, "[") || strings.HasPrefix(line, " [") || strings.HasPrefix(line, " [") || line == "" || ignoreLineFlag) { + flag := strings.Fields(line)[0] + if flag == "snapshot" { + flag = strconv.Itoa(cacheFlag) + "-" + flag + } else if flag == "corsdomain" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "vhosts" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "enabled" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "host" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "api" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "port" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "prefix" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "database" { + flag = strconv.Itoa(databaseFlag) + "-" + flag + } else if flag == "password" { + flag = strconv.Itoa(passwordFlag) + "-" + flag + } + + if flag != "static-nodes" && flag != "trusted-nodes" { + flag = nameTagMap[flag] + + tempFlag := false + + for _, val := range updatedArgs { + if strings.Contains(val, flag) && (strings.Contains(val, "-") || strings.Contains(val, "--")) { + tempFlag = true + } + } + + if !tempFlag || flag == "" { + newLines[i] = "# " + line + } + } + } + } + + output := strings.Join(newLines, "\n") + + err = os.WriteFile(path, []byte(output), 0600) + if err != nil { + log.Fatalln(err) + } +} + +func checkBoolFlags(val string) bool { + returnFlag := false + + if strings.Contains(val, "=") { + val = strings.Split(val, "=")[0] + } + + for _, flag := range currentBoolFlags { + if val == "-"+flag || val == "--"+flag { + returnFlag = true + } + } + + return returnFlag +} + +func beautifyArgs(args []string) ([]string, []string) { + newArgs := []string{} + + ignoreForNow := []string{} + + temp := []string{} + + for _, val := range args { + // nolint: nestif + if !(checkBoolFlags(val)) { + if strings.HasPrefix(val, "-") { + if strings.Contains(val, "=") { + temparr := strings.Split(val, "=") + newArgs = append(newArgs, temparr...) + } else { + newArgs = append(newArgs, val) + } + } else { + newArgs = append(newArgs, val) + } + } else { + ignoreForNow = append(ignoreForNow, val) + } + } + + for j, val := range newArgs { + if val == "-unlock" || val == "--unlock" { + temp = append(temp, "--miner.etherbase") + temp = append(temp, newArgs[j+1]) + } + } + + newArgs = append(newArgs, temp...) + + return newArgs, ignoreForNow +} + +func main() { + const notYet = "notYet" + + temp := os.Args[1] + args := os.Args[2:] + + args, ignoreForNow := beautifyArgs(args) + + c := server.Command{} + flags := c.Flags() + allFlags := flags.GetAllFlags() + flagsToCheck := getFlagsToCheck(args) + + if temp == notYet { + getStaticTrustedNodes(args) + } + + outOfDateFlags := checkFlag(allFlags, flagsToCheck) + updatedArgs := updateArgsClean(args, outOfDateFlags) + updatedArgs = updateArgsAdd(updatedArgs) + updatedArgs = handlePrometheus(args, updatedArgs) + + if temp == notYet { + updatedArgs = append(updatedArgs, ignoreForNow...) + dumpFlags(updatedArgs) + } + + if temp != notYet { + updatedArgs = append(updatedArgs, ignoreForNow...) + commentFlags(temp, updatedArgs) + } +} diff --git a/scripts/getconfig.sh b/scripts/getconfig.sh new file mode 100755 index 0000000000..26d5d0138c --- /dev/null +++ b/scripts/getconfig.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env sh + +# Instructions: +# Execute `./getconfig.sh`, and follow the instructions displayed on the terminal +# The `*-config.toml` file will be created in the same directory as start.sh +# It is recommended to check the flags generated in config.toml + + +read -p "* Path to start.sh: " startPath +# check if start.sh is present +if [[ ! -f $startPath ]] +then + echo "Error: start.sh do not exist." + exit 1 +fi +read -p "* Your validator address (e.g. 0xca67a8D767e45056DC92384b488E9Af654d78DE2), or press Enter to skip if running a sentry node: " ADD + +echo "\nThank you, your inputs are:" +echo "Path to start.sh: "$startPath +echo "Address: "$ADD + +confPath=${startPath%.sh}"-config.toml" +echo "Path to the config file: "$confPath +# check if config.toml is present +if [[ -f $confPath ]] +then + echo "WARN: config.toml exists, data will be overwritten." +fi + +tmpDir="$(mktemp -d -t ./temp-dir-XXXXXXXXXXX || oops "Can't create temporary directory")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +# SHA1 hash of `tempStart` -> `3305fe263dd4a999d58f96deb064e21bb70123d9` +sed 's/bor --/go run getconfig.go notYet --/g' $startPath > $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +chmod +x $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +$tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $ADD +rm $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh + + +sed -i '' "s%*%'*'%g" ./temp + +# read the flags from `./temp` +dumpconfigflags=$(head -1 ./temp) + +# run the dumpconfig command with the flags from `./temp` +command="bor dumpconfig "$dumpconfigflags" > "$confPath +bash -c "$command" + +rm ./temp + +if [[ -f ./tempStaticNodes.json ]] +then + echo "JSON StaticNodes found" + staticnodesjson=$(head -1 ./tempStaticNodes.json) + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodesjson}\"\]%" $confPath + rm ./tempStaticNodes.json +elif [[ -f ./tempStaticNodes.toml ]] +then + echo "TOML StaticNodes found" + staticnodestoml=$(head -1 ./tempStaticNodes.toml) + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodestoml}\"\]%" $confPath + rm ./tempStaticNodes.toml +else + echo "neither JSON nor TOML StaticNodes found" +fi + +if [[ -f ./tempTrustedNodes.toml ]] +then + echo "TOML TrustedNodes found" + trustednodestoml=$(head -1 ./tempTrustedNodes.toml) + sed -i '' "s%trusted-nodes = \[\]%trusted-nodes = \[\"${trustednodestoml}\"\]%" $confPath + rm ./tempTrustedNodes.toml +else + echo "neither JSON nor TOML TrustedNodes found" +fi + +# comment flags in $configPath that were not passed through $startPath +# SHA1 hash of `tempStart` -> `3305fe263dd4a999d58f96deb064e21bb70123d9` +sed "s%bor --%go run getconfig.go ${confPath} --%" $startPath > $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +chmod +x $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +$tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $ADD +rm $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh + +exit 0 From c7246711725de10452d436b3f86d26f28821acfb Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Thu, 25 Aug 2022 23:13:22 +0530 Subject: [PATCH 24/70] rm: snapshot found log in bor consensus (#504) --- consensus/bor/bor.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 826a45dae2..f140146976 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -535,8 +535,6 @@ func (c *Bor) snapshot(chain consensus.ChainHeaderReader, number uint64, hash co number, hash = number-1, header.ParentHash } - log.Info("Snapshot has been found in", "headers depth", len(headers)) - // check if snapshot is nil if snap == nil { return nil, fmt.Errorf("Unknown error while retrieving snapshot at block number %v", number) From 1a515607262cd327ae7345671ba316e98f7ebe74 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 4 Oct 2022 21:54:47 +0530 Subject: [PATCH 25/70] make script OS independent (#538) --- scripts/getconfig.sh | 43 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/scripts/getconfig.sh b/scripts/getconfig.sh index 26d5d0138c..943d540a88 100755 --- a/scripts/getconfig.sh +++ b/scripts/getconfig.sh @@ -1,10 +1,19 @@ #!/usr/bin/env sh +set -e # Instructions: # Execute `./getconfig.sh`, and follow the instructions displayed on the terminal # The `*-config.toml` file will be created in the same directory as start.sh # It is recommended to check the flags generated in config.toml +# Some checks to make commands OS independent +OS="$(uname -s)" +MKTEMPOPTION= +SEDOPTION= ## Not used as of now (TODO) +shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then + SEDOPTION="''" + MKTEMPOPTION="-t" +fi read -p "* Path to start.sh: " startPath # check if start.sh is present @@ -15,7 +24,7 @@ then fi read -p "* Your validator address (e.g. 0xca67a8D767e45056DC92384b488E9Af654d78DE2), or press Enter to skip if running a sentry node: " ADD -echo "\nThank you, your inputs are:" +printf "\nThank you, your inputs are:\n" echo "Path to start.sh: "$startPath echo "Address: "$ADD @@ -26,8 +35,9 @@ if [[ -f $confPath ]] then echo "WARN: config.toml exists, data will be overwritten." fi +printf "\n" -tmpDir="$(mktemp -d -t ./temp-dir-XXXXXXXXXXX || oops "Can't create temporary directory")" +tmpDir="$(mktemp -d $MKTEMPOPTION ./temp-dir-XXXXXXXXXXX || oops "Can't create temporary directory")" cleanup() { rm -rf "$tmpDir" } @@ -39,8 +49,11 @@ chmod +x $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $ADD rm $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh - -sed -i '' "s%*%'*'%g" ./temp +shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then + sed -i '' "s%*%'*'%g" ./temp +else + sed -i "s%*%'*'%g" ./temp +fi # read the flags from `./temp` dumpconfigflags=$(head -1 ./temp) @@ -51,17 +64,27 @@ bash -c "$command" rm ./temp +printf "\n" + if [[ -f ./tempStaticNodes.json ]] then echo "JSON StaticNodes found" staticnodesjson=$(head -1 ./tempStaticNodes.json) - sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodesjson}\"\]%" $confPath + shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodesjson}\"\]%" $confPath + else + sed -i "s%static-nodes = \[\]%static-nodes = \[\"${staticnodesjson}\"\]%" $confPath + fi rm ./tempStaticNodes.json elif [[ -f ./tempStaticNodes.toml ]] then echo "TOML StaticNodes found" staticnodestoml=$(head -1 ./tempStaticNodes.toml) - sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodestoml}\"\]%" $confPath + shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodestoml}\"\]%" $confPath + else + sed -i "s%static-nodes = \[\]%static-nodes = \[\"${staticnodestoml}\"\]%" $confPath + fi rm ./tempStaticNodes.toml else echo "neither JSON nor TOML StaticNodes found" @@ -71,12 +94,18 @@ if [[ -f ./tempTrustedNodes.toml ]] then echo "TOML TrustedNodes found" trustednodestoml=$(head -1 ./tempTrustedNodes.toml) - sed -i '' "s%trusted-nodes = \[\]%trusted-nodes = \[\"${trustednodestoml}\"\]%" $confPath + shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then + sed -i '' "s%trusted-nodes = \[\]%trusted-nodes = \[\"${trustednodestoml}\"\]%" $confPath + else + sed -i "s%trusted-nodes = \[\]%trusted-nodes = \[\"${trustednodestoml}\"\]%" $confPath + fi rm ./tempTrustedNodes.toml else echo "neither JSON nor TOML TrustedNodes found" fi +printf "\n" + # comment flags in $configPath that were not passed through $startPath # SHA1 hash of `tempStart` -> `3305fe263dd4a999d58f96deb064e21bb70123d9` sed "s%bor --%go run getconfig.go ${confPath} --%" $startPath > $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh From 083c3fead17e2317b45925d3547cc9497d2313e5 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Wed, 5 Oct 2022 23:38:19 +0530 Subject: [PATCH 26/70] internal/cli: add support for bor.logs flag in new-cli (#541) * add support for bor.logs flag in new-cli * handle bor.withoutheimdall flag commenting in conversion script --- builder/files/config.toml | 3 ++- docs/cli/removedb.md | 4 +++- docs/cli/server.md | 10 +++++++--- internal/cli/server/config.go | 8 +++++++- internal/cli/server/flags.go | 10 ++++++++-- scripts/getconfig.go | 4 +++- 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index ce9fd782d0..0d01f85d71 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -10,6 +10,7 @@ datadir = "/var/lib/bor/data" syncmode = "full" # gcmode = "full" # snapshot = true +# "bor.logs" = false # ethstats = "" # ["eth.requiredblocks"] @@ -134,4 +135,4 @@ syncmode = "full" # [developer] # dev = false - # period = 0 \ No newline at end of file + # period = 0 diff --git a/docs/cli/removedb.md b/docs/cli/removedb.md index 3c6e84f1d6..473d47ecef 100644 --- a/docs/cli/removedb.md +++ b/docs/cli/removedb.md @@ -4,4 +4,6 @@ The ```bor removedb``` command will remove the blockchain and state databases at ## Options -- ```datadir```: Path of the data directory to store information +- ```address```: Address of the grpc endpoint + +- ```datadir```: Path of the data directory to store information \ No newline at end of file diff --git a/docs/cli/server.md b/docs/cli/server.md index ac43555275..5fe440b5fd 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -16,18 +16,22 @@ The ```bor server``` command runs the Bor client. - ```config```: File for the config file -- ```syncmode```: Blockchain sync mode ("fast", "full", or "snap") +- ```syncmode```: Blockchain sync mode (only "full" sync supported) - ```gcmode```: Blockchain garbage collection mode ("full", "archive") -- ```requiredblocks```: Comma separated block number-to-hash mappings to enforce (=) +- ```eth.requiredblocks```: Comma separated block number-to-hash mappings to require for peering (=) -- ```snapshot```: Disables/Enables the snapshot-database mode (default = true) +- ```snapshot```: Enables the snapshot-database mode (default = true) + +- ```bor.logs```: Enables bor log retrieval (default = false) - ```bor.heimdall```: URL of Heimdall service - ```bor.withoutheimdall```: Run without Heimdall service (for testing purpose) +- ```bor.heimdallgRPC```: Address of Heimdall gRPC service + - ```ethstats```: Reporting URL of a ethstats service (nodename:secret@host:port) - ```gpo.blocks```: Number of recent blocks to check for gas prices diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index a4f642a6e3..c9e770c5ba 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -62,9 +62,12 @@ type Config struct { // GcMode selects the garbage collection mode for the trie GcMode string `hcl:"gcmode,optional" toml:"gcmode,optional"` - // Snapshot disables/enables the snapshot database mode + // Snapshot enables the snapshot database mode Snapshot bool `hcl:"snapshot,optional" toml:"snapshot,optional"` + // BorLogs enables bor log retrieval + BorLogs bool `hcl:"bor.logs,optional" toml:"bor.logs,optional"` + // Ethstats is the address of the ethstats server to send telemetry Ethstats string `hcl:"ethstats,optional" toml:"ethstats,optional"` @@ -416,6 +419,7 @@ func DefaultConfig() *Config { SyncMode: "full", GcMode: "full", Snapshot: true, + BorLogs: false, TxPool: &TxPoolConfig{ Locals: []string{}, NoLocals: false, @@ -645,6 +649,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* n.NetworkId = c.chain.NetworkId n.Genesis = c.chain.Genesis } + n.HeimdallURL = c.Heimdall.URL n.WithoutHeimdall = c.Heimdall.Without @@ -876,6 +881,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* } } + n.BorLogs = c.BorLogs n.DatabaseHandles = dbHandles return &n, nil diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 835ff0ab95..ec74f89991 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -45,7 +45,7 @@ func (c *Command) Flags() *flagset.Flagset { }) f.StringFlag(&flagset.StringFlag{ Name: "syncmode", - Usage: `Blockchain sync mode ("fast", "full", or "snap")`, + Usage: `Blockchain sync mode (only "full" sync supported)`, Value: &c.cliConfig.SyncMode, Default: c.cliConfig.SyncMode, }) @@ -62,10 +62,16 @@ func (c *Command) Flags() *flagset.Flagset { }) f.BoolFlag(&flagset.BoolFlag{ Name: "snapshot", - Usage: `Disables/Enables the snapshot-database mode (default = true)`, + Usage: `Enables the snapshot-database mode (default = true)`, Value: &c.cliConfig.Snapshot, Default: c.cliConfig.Snapshot, }) + f.BoolFlag(&flagset.BoolFlag{ + Name: "bor.logs", + Usage: `Enables bor log retrieval (default = false)`, + Value: &c.cliConfig.BorLogs, + Default: c.cliConfig.BorLogs, + }) // heimdall f.StringFlag(&flagset.StringFlag{ diff --git a/scripts/getconfig.go b/scripts/getconfig.go index 689ed68fbf..136b69ecab 100644 --- a/scripts/getconfig.go +++ b/scripts/getconfig.go @@ -109,8 +109,9 @@ var nameTagMap = map[string]string{ "gcmode": "gcmode", "eth.requiredblocks": "eth.requiredblocks", "0-snapshot": "snapshot", + "\"bor.logs\"": "bor.logs", "url": "bor.heimdall", - "bor.without": "bor.withoutheimdall", + "\"bor.without\"": "bor.withoutheimdall", "grpc-address": "bor.heimdallgRPC", "locals": "txpool.locals", "nolocals": "txpool.nolocals", @@ -231,6 +232,7 @@ var replacedFlagsMapFlag = map[string]string{ var currentBoolFlags = []string{ "snapshot", + "bor.logs", "bor.withoutheimdall", "txpool.nolocals", "mine", From 027025852edd985988a6be99a8cf78f332d78679 Mon Sep 17 00:00:00 2001 From: Evgeny Danienko <6655321@bk.ru> Date: Wed, 12 Oct 2022 11:35:00 +0400 Subject: [PATCH 27/70] commit logs --- miner/worker.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/miner/worker.go b/miner/worker.go index 797e7ea980..2c8cb3f790 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -918,6 +918,19 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP } var coalescedLogs []*types.Log + initialGasLimit := env.gasPool.Gas() + initialTxs := txs.GetTxs() + var breakCause string + + defer func() { + log.Warn("commitTransactions-stats", + "initialTxsCount", initialTxs, + "initialGasLimit", initialGasLimit, + "resultTxsCount", txs.GetTxs(), + "resultGapPool", env.gasPool.Gas(), + "exitCause", breakCause) + }() + for { // In the following three cases, we will interrupt the execution of the transaction. // (1) new head block event arrival, the interrupt signal is 1 @@ -937,16 +950,19 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP inc: true, } } + breakCause = "interrupt" return atomic.LoadInt32(interrupt) == commitInterruptNewHead } // If we don't have enough gas for any further transactions then we're done if env.gasPool.Gas() < params.TxGas { log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) + breakCause = "Not enough gas for further transactions" break } // Retrieve the next transaction and abort if all done tx := txs.Peek() if tx == nil { + breakCause = "all transactions has been included" break } // Error may be ignored here. The error has already been checked From cb12a6e4d6b93f86777128463738953649bee929 Mon Sep 17 00:00:00 2001 From: Jerry Date: Wed, 19 Oct 2022 14:55:31 -0700 Subject: [PATCH 28/70] Force load default tracers --- internal/cli/server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/cli/server/server.go b/internal/cli/server/server.go index adacb44ce7..6758d74312 100644 --- a/internal/cli/server/server.go +++ b/internal/cli/server/server.go @@ -36,6 +36,10 @@ import ( "github.com/ethereum/go-ethereum/metrics/influxdb" "github.com/ethereum/go-ethereum/metrics/prometheus" "github.com/ethereum/go-ethereum/node" + + // Force-load the tracer engines to trigger registration + _ "github.com/ethereum/go-ethereum/eth/tracers/js" + _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) type Server struct { From a977a76f97251d0a902921dd3a93610d969bb645 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Fri, 21 Oct 2022 02:15:42 +0530 Subject: [PATCH 29/70] metrics: handle values from config file (#565) * metrics: handle metrics flag from config in metrics.init() * internal/cli/server: log for metrics enabling and misconfiguration --- internal/cli/server/server.go | 10 +++++++ metrics/metrics.go | 53 ++++++++++++++++++++++++++++++++--- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/internal/cli/server/server.go b/internal/cli/server/server.go index 6758d74312..1346fe613a 100644 --- a/internal/cli/server/server.go +++ b/internal/cli/server/server.go @@ -233,6 +233,12 @@ func (s *Server) Stop() { } func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error { + // Check the global metrics if they're matching with the provided config + if metrics.Enabled != config.Enabled || metrics.EnabledExpensive != config.Expensive { + log.Warn("Metric misconfiguration, some of them might not be visible") + } + + // Update the values anyways (for services which don't need immediate attention) metrics.Enabled = config.Enabled metrics.EnabledExpensive = config.Expensive @@ -243,6 +249,10 @@ func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error log.Info("Enabling metrics collection") + if metrics.EnabledExpensive { + log.Info("Enabling expensive metrics collection") + } + // influxdb if v1Enabled, v2Enabled := config.InfluxDB.V1Enabled, config.InfluxDB.V2Enabled; v1Enabled || v2Enabled { if v1Enabled && v2Enabled { diff --git a/metrics/metrics.go b/metrics/metrics.go index 747d6471a7..1d0133e850 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/ethereum/go-ethereum/log" + "github.com/BurntSushi/toml" ) // Enabled is checked by the constructor functions for all of the @@ -32,26 +32,71 @@ var enablerFlags = []string{"metrics"} // expensiveEnablerFlags is the CLI flag names to use to enable metrics collections. var expensiveEnablerFlags = []string{"metrics.expensive"} +// configFlag is the CLI flag name to use to start node by providing a toml based config +var configFlag = "config" + // Init enables or disables the metrics system. Since we need this to run before // any other code gets to create meters and timers, we'll actually do an ugly hack // and peek into the command line args for the metrics flag. func init() { - for _, arg := range os.Args { + var configFile string + + for i := 0; i < len(os.Args); i++ { + arg := os.Args[i] + flag := strings.TrimLeft(arg, "-") + // check for existence of `config` flag + if flag == configFlag && i < len(os.Args)-1 { + configFile = strings.TrimLeft(os.Args[i+1], "-") // find the value of flag + } + for _, enabler := range enablerFlags { if !Enabled && flag == enabler { - log.Info("Enabling metrics collection") Enabled = true } } + for _, enabler := range expensiveEnablerFlags { if !EnabledExpensive && flag == enabler { - log.Info("Enabling expensive metrics collection") EnabledExpensive = true } } } + + // Update the global metrics value, if they're provided in the config file + updateMetricsFromConfig(configFile) +} + +func updateMetricsFromConfig(path string) { + // Don't act upon any errors here. They're already taken into + // consideration when the toml config file will be parsed in the cli. + data, err := os.ReadFile(path) + tomlData := string(data) + + if err != nil { + return + } + + // Create a minimal config to decode + type TelemetryConfig struct { + Enabled bool `hcl:"metrics,optional" toml:"metrics,optional"` + Expensive bool `hcl:"expensive,optional" toml:"expensive,optional"` + } + + type CliConfig struct { + Telemetry *TelemetryConfig `hcl:"telemetry,block" toml:"telemetry,block"` + } + + conf := &CliConfig{} + + if _, err := toml.Decode(tomlData, &conf); err != nil || conf == nil { + return + } + + // We have the values now, update them + Enabled = conf.Telemetry.Enabled + EnabledExpensive = conf.Telemetry.Expensive } // CollectProcessMetrics periodically collects various metrics about the running From 5c5c9924405721b03f240ae994520fe94fec5b6e Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 21 Oct 2022 08:54:38 +0530 Subject: [PATCH 30/70] Changed default value of maxpeers from 200 to 50, update docs (#555) * changed default of maxpeers from 200 to 50 * docs: update additional notes for new-cli * docs: update additional notes for new-cli * small bug fix in the script to fix shopt issue Co-authored-by: Manav Darji --- cmd/geth/config.go | 4 ++-- docs/README.md | 18 ++++++++---------- docs/config.md | 2 +- internal/cli/server/config.go | 2 +- scripts/getconfig.sh | 2 +- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 08b76f83da..101f4b2de6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -329,7 +329,7 @@ func setDefaultMumbaiGethConfig(ctx *cli.Context, config *gethConfig) { config.Eth.TxPool.AccountQueue = 64 config.Eth.TxPool.GlobalQueue = 131072 config.Eth.TxPool.Lifetime = 90 * time.Minute - config.Node.P2P.MaxPeers = 200 + config.Node.P2P.MaxPeers = 50 config.Metrics.Enabled = true // --pprof is enabled in 'internal/debug/flags.go' } @@ -352,7 +352,7 @@ func setDefaultBorMainnetGethConfig(ctx *cli.Context, config *gethConfig) { config.Eth.TxPool.AccountQueue = 64 config.Eth.TxPool.GlobalQueue = 131072 config.Eth.TxPool.Lifetime = 90 * time.Minute - config.Node.P2P.MaxPeers = 200 + config.Node.P2P.MaxPeers = 50 config.Metrics.Enabled = true // --pprof is enabled in 'internal/debug/flags.go' } diff --git a/docs/README.md b/docs/README.md index 95ba38b0da..5ebdbd7e26 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,18 +5,16 @@ - [Configuration file](./config.md) -## Deprecation notes +## Additional notes - The new entrypoint to run the Bor client is ```server```. -``` -$ bor server -``` + ``` + $ bor server + ``` -- Toml files to configure nodes are being deprecated. Currently, we only allow for static and trusted nodes to be configured using toml files. +- Toml files used earlier just to configure static/trusted nodes are being deprecated. Instead, a toml file now can be used instead of flags and can contain all configuration for the node to run. The link to a sample config file is given above. To simply run bor with a configuration file, the following command can be used. -``` -$ bor server --config ./legacy.toml -``` - -- ```Admin```, ```Personal``` and account related endpoints in ```Eth``` are being removed from the JsonRPC interface. Some of this functionality will be moved to the new GRPC server for operational tasks. + ``` + $ bor server --config + ``` diff --git a/docs/config.md b/docs/config.md index aebd9e12b9..57f4c25fef 100644 --- a/docs/config.md +++ b/docs/config.md @@ -22,7 +22,7 @@ ethstats = "" ["eth.requiredblocks"] [p2p] -maxpeers = 30 +maxpeers = 50 maxpendpeers = 50 bind = "0.0.0.0" port = 30303 diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index c9e770c5ba..86b611ac78 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -396,7 +396,7 @@ func DefaultConfig() *Config { LogLevel: "INFO", DataDir: DefaultDataDir(), P2P: &P2PConfig{ - MaxPeers: 30, + MaxPeers: 50, MaxPendPeers: 50, Bind: "0.0.0.0", Port: 30303, diff --git a/scripts/getconfig.sh b/scripts/getconfig.sh index 943d540a88..a2971c4f12 100755 --- a/scripts/getconfig.sh +++ b/scripts/getconfig.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/bin/bash set -e # Instructions: From b8c3b0043c1c7ad4f57b956e4ebcba30e9ee4bf8 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Wed, 12 Oct 2022 20:06:00 +0530 Subject: [PATCH 31/70] add : Add state sync transaction to debug_traceBlock --- consensus/bor/statefull/processor.go | 52 +++++++-- core/vm/interface.go | 2 + eth/tracers/api.go | 159 +++++++++++++++++++++------ eth/tracers/api_test.go | 11 +- 4 files changed, 178 insertions(+), 46 deletions(-) diff --git a/consensus/bor/statefull/processor.go b/consensus/bor/statefull/processor.go index e30fb4fd21..192ea5fa38 100644 --- a/consensus/bor/statefull/processor.go +++ b/consensus/bor/statefull/processor.go @@ -30,22 +30,22 @@ func (c ChainContext) GetHeader(hash common.Hash, number uint64) *types.Header { } // callmsg implements core.Message to allow passing it as a transaction simulator. -type callmsg struct { +type Callmsg struct { ethereum.CallMsg } -func (m callmsg) From() common.Address { return m.CallMsg.From } -func (m callmsg) Nonce() uint64 { return 0 } -func (m callmsg) CheckNonce() bool { return false } -func (m callmsg) To() *common.Address { return m.CallMsg.To } -func (m callmsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } -func (m callmsg) Gas() uint64 { return m.CallMsg.Gas } -func (m callmsg) Value() *big.Int { return m.CallMsg.Value } -func (m callmsg) Data() []byte { return m.CallMsg.Data } +func (m Callmsg) From() common.Address { return m.CallMsg.From } +func (m Callmsg) Nonce() uint64 { return 0 } +func (m Callmsg) CheckNonce() bool { return false } +func (m Callmsg) To() *common.Address { return m.CallMsg.To } +func (m Callmsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } +func (m Callmsg) Gas() uint64 { return m.CallMsg.Gas } +func (m Callmsg) Value() *big.Int { return m.CallMsg.Value } +func (m Callmsg) Data() []byte { return m.CallMsg.Data } // get system message -func GetSystemMessage(toAddress common.Address, data []byte) callmsg { - return callmsg{ +func GetSystemMessage(toAddress common.Address, data []byte) Callmsg { + return Callmsg{ ethereum.CallMsg{ From: systemAddress, Gas: math.MaxUint64 / 2, @@ -59,7 +59,7 @@ func GetSystemMessage(toAddress common.Address, data []byte) callmsg { // apply message func ApplyMessage( - msg callmsg, + msg Callmsg, state *state.StateDB, header *types.Header, chainConfig *params.ChainConfig, @@ -90,4 +90,32 @@ func ApplyMessage( gasUsed := initialGas - gasLeft return gasUsed, nil + +} + +func ApplyBorMessage(vmenv vm.EVM, msg Callmsg) (*core.ExecutionResult, error) { + + initialGas := msg.Gas() + + // Apply the transaction to the current state (included in the env) + ret, gasLeft, err := vmenv.Call( + vm.AccountRef(msg.From()), + *msg.To(), + msg.Data(), + msg.Gas(), + msg.Value(), + ) + // Update the state with pending changes + if err != nil { + vmenv.StateDB.Finalise(true) + } + + gasUsed := initialGas - gasLeft + + return &core.ExecutionResult{ + UsedGas: gasUsed, + Err: err, + ReturnData: ret, + }, nil + } diff --git a/core/vm/interface.go b/core/vm/interface.go index ad9b05d666..1064adf590 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -74,6 +74,8 @@ type StateDB interface { AddPreimage(common.Hash, []byte) ForEachStorage(common.Address, func(common.Hash, common.Hash) bool) error + + Finalise(bool) } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 08c17601e4..12b1411ae7 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -28,9 +28,11 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/consensus/bor/statefull" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -80,6 +82,9 @@ type Backend interface { // so this method should be called with the parent. StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, checkLive, preferDisk bool) (*state.StateDB, error) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) + + // Bor related APIs + GetBorBlockTransactionWithBlockHash(ctx context.Context, txHash common.Hash, blockHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) } // API is the collection of tracing APIs exposed over the private debugging endpoint. @@ -164,6 +169,25 @@ func (api *API) blockByNumberAndHash(ctx context.Context, number rpc.BlockNumber return api.blockByHash(ctx, hash) } +// returns block transactions along with state-sync transaction if present +func (api *API) getAllBlockTransactions(ctx context.Context, block *types.Block) (types.Transactions, bool) { + txs := block.Transactions() + + stateSyncPresent := false + + borReceipt := rawdb.ReadBorReceipt(api.backend.ChainDb(), block.Hash(), block.NumberU64()) + if borReceipt != nil { + txHash := types.GetDerivedBorTxHash(types.BorReceiptKey(block.Number().Uint64(), block.Hash())) + if txHash != (common.Hash{}) { + borTx, _, _, _, _ := api.backend.GetBorBlockTransactionWithBlockHash(ctx, txHash, block.Hash()) + txs = append(txs, borTx) + stateSyncPresent = true + } + } + + return txs, stateSyncPresent +} + // TraceConfig holds extra parameters to trace functions. type TraceConfig struct { *logger.Config @@ -274,19 +298,30 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config signer := types.MakeSigner(api.backend.ChainConfig(), task.block.Number()) blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(localctx), nil) // Trace all the transactions contained within - for i, tx := range task.block.Transactions() { + txs, stateSyncPresent := api.getAllBlockTransactions(ctx, task.block) + for i, tx := range txs { msg, _ := tx.AsMessage(signer, task.block.BaseFee()) txctx := &Context{ BlockHash: task.block.Hash(), TxIndex: i, TxHash: tx.Hash(), } - res, err := api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) + + var res interface{} + var err error + + if stateSyncPresent && i == len(txs)-1 { + res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config, true) + } else { + res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config, false) + } + if err != nil { task.results[i] = &txTraceResult{Error: err.Error()} log.Warn("Tracing failed", "hash", tx.Hash(), "block", task.block.NumberU64(), "err", err) break } + // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect task.statedb.Finalise(api.backend.ChainConfig().IsEIP158(task.block.Number())) task.results[i] = &txTraceResult{Result: res} @@ -492,6 +527,23 @@ func (api *API) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, return api.standardTraceBlockToFile(ctx, block, config) } +func prepareCallMessage(msg core.Message) statefull.Callmsg { + + return statefull.Callmsg{ + ethereum.CallMsg{ + From: msg.From(), + To: msg.To(), + Gas: msg.Gas(), + GasPrice: msg.GasPrice(), + GasFeeCap: msg.GasFeeCap(), + GasTipCap: msg.GasTipCap(), + Value: msg.Value(), + Data: msg.Data(), + AccessList: msg.AccessList(), + }} + +} + // IntermediateRoots executes a block (bad- or canon- or side-), and returns a list // of intermediate roots: the stateroot after each transaction. func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config *TraceConfig) ([]common.Hash, error) { @@ -525,23 +577,42 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) deleteEmptyObjects = chainConfig.IsEIP158(block.Number()) ) - for i, tx := range block.Transactions() { + txs, stateSyncPresent := api.getAllBlockTransactions(ctx, block) + for i, tx := range txs { var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) ) statedb.Prepare(tx.Hash(), i) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { - log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) - // We intentionally don't return the error here: if we do, then the RPC server will not - // return the roots. Most likely, the caller already knows that a certain transaction fails to - // be included, but still want the intermediate roots that led to that point. - // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be - // executable. - // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. - return roots, nil + if stateSyncPresent && i == len(txs)-1 { + callmsg := prepareCallMessage(msg) + + if _, err := statefull.ApplyMessage(ctx, callmsg, statedb, block.Header(), api.backend.ChainConfig(), api.chainContext(ctx)); err != nil { + log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) + // We intentionally don't return the error here: if we do, then the RPC server will not + // return the roots. Most likely, the caller already knows that a certain transaction fails to + // be included, but still want the intermediate roots that led to that point. + // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be + // executable. + // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. + return roots, nil + } + + } else { + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { + log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) + // We intentionally don't return the error here: if we do, then the RPC server will not + // return the roots. Most likely, the caller already knows that a certain transaction fails to + // be included, but still want the intermediate roots that led to that point. + // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be + // executable. + // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. + return roots, nil + } + } + // calling IntermediateRoot will internally call Finalize on the state // so any modifications are written to the trie roots = append(roots, statedb.IntermediateRoot(deleteEmptyObjects)) @@ -581,9 +652,9 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac } // Execute all the transaction contained within the block concurrently var ( - signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) - txs = block.Transactions() - results = make([]*txTraceResult, len(txs)) + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + txs, stateSyncPresent = api.getAllBlockTransactions(ctx, block) + results = make([]*txTraceResult, len(txs)) pend = new(sync.WaitGroup) jobs = make(chan *txTraceTask, len(txs)) @@ -606,7 +677,13 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac TxIndex: task.index, TxHash: txs[task.index].Hash(), } - res, err := api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) + var res interface{} + var err error + if stateSyncPresent && task.index == len(txs)-1 { + res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config, true) + } else { + res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config, false) + } if err != nil { results[task.index] = &txTraceResult{Error: err.Error()} continue @@ -650,7 +727,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { // If we're tracing a single transaction, make sure it's present if config != nil && config.TxHash != (common.Hash{}) { - if !containsTx(block, config.TxHash) { + if !api.containsTx(ctx, block, config.TxHash) { return nil, fmt.Errorf("transaction %#x not found in block", config.TxHash) } } @@ -705,7 +782,8 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block canon = false } } - for i, tx := range block.Transactions() { + txs, stateSyncPresent := api.getAllBlockTransactions(ctx, block) + for i, tx := range txs { // Prepare the trasaction for un-traced execution var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) @@ -739,10 +817,19 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) statedb.Prepare(tx.Hash(), i) - _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) - if writer != nil { - writer.Flush() + if stateSyncPresent && i == len(txs)-1 { + callmsg := prepareCallMessage(msg) + _, err = statefull.ApplyBorMessage(*vmenv, callmsg) + if writer != nil { + writer.Flush() + } + } else { + _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) + if writer != nil { + writer.Flush() + } } + if dump != nil { dump.Close() log.Info("Wrote standard trace", "file", dump.Name()) @@ -764,8 +851,9 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // containsTx reports whether the transaction with a certain hash // is contained within the specified block. -func containsTx(block *types.Block, hash common.Hash) bool { - for _, tx := range block.Transactions() { +func (api *API) containsTx(ctx context.Context, block *types.Block, hash common.Hash) bool { + txs, _ := api.getAllBlockTransactions(ctx, block) + for _, tx := range txs { if tx.Hash() == hash { return true } @@ -775,7 +863,7 @@ func containsTx(block *types.Block, hash common.Hash) bool { // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { +func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig, borTx bool) (interface{}, error) { tx, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) if tx == nil { // For BorTransaction, there will be no trace available @@ -811,14 +899,14 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * TxIndex: int(index), TxHash: hash, } - return api.traceTx(ctx, msg, txctx, vmctx, statedb, config) + return api.traceTx(ctx, msg, txctx, vmctx, statedb, config, borTx) } // TraceCall lets you trace a given eth_call. It collects the structured logs // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. // You can provide -2 as a block number to trace on top of the pending block. -func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { +func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig, borTx bool) (interface{}, error) { // Try to retrieve the specified block var ( err error @@ -865,13 +953,13 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc Reexec: config.Reexec, } } - return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig) + return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig, borTx) } // traceTx configures a new tracer according to the provided configuration, and // executes the given message in the provided environment. The return value will // be tracer dependent. -func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { +func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig, borTx bool) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( tracer vm.EVMLogger @@ -911,9 +999,18 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Call Prepare to clear out the statedb access list statedb.Prepare(txctx.TxHash, txctx.TxIndex) - result, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) - if err != nil { - return nil, fmt.Errorf("tracing failed: %w", err) + var result *core.ExecutionResult + if borTx { + callmsg := prepareCallMessage(message) + if result, err = statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { + return nil, fmt.Errorf("tracing failed: %w", err) + } + + } else { + result, err = core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) + if err != nil { + return nil, fmt.Errorf("tracing failed: %w", err) + } } // Depending on the tracer type, format and return the output. diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index a3c0a72494..32c36f16ee 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -176,6 +176,11 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return nil, vm.BlockContext{}, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } +func (b *testBackend) GetBorBlockTransactionWithBlockHash(ctx context.Context, txHash common.Hash, blockHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + tx, blockHash, blockNumber, index := rawdb.ReadBorTransactionWithBlockHash(b.ChainDb(), txHash, blockHash) + return tx, blockHash, blockNumber, index, nil +} + func TestTraceCall(t *testing.T) { t.Parallel() @@ -285,7 +290,7 @@ func TestTraceCall(t *testing.T) { }, } for _, testspec := range testSuite { - result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) + result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config, false) if testspec.expectErr != nil { if err == nil { t.Errorf("Expect error %v, get nothing", testspec.expectErr) @@ -325,7 +330,7 @@ func TestTraceTransaction(t *testing.T) { b.AddTx(tx) target = tx.Hash() })) - result, err := api.TraceTransaction(context.Background(), target, nil) + result, err := api.TraceTransaction(context.Background(), target, nil, false) if err != nil { t.Errorf("Failed to trace transaction %v", err) } @@ -508,7 +513,7 @@ func TestTracingWithOverrides(t *testing.T) { }, } for i, tc := range testSuite { - result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config) + result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config, false) if tc.expectErr != nil { if err == nil { t.Errorf("test %d: want error %v, have nothing", i, tc.expectErr) From ee18b3cd18bb5ec6d435b761f68c34936dfe0c39 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Thu, 13 Oct 2022 15:36:21 +0530 Subject: [PATCH 32/70] chg : major fix --- eth/tracers/api.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 12b1411ae7..c6398d1a8a 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -702,11 +702,22 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // Generate the next state snapshot fast without tracing msg, _ := tx.AsMessage(signer, block.BaseFee()) statedb.Prepare(tx.Hash(), i) + vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) - if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { - failed = err - break + + if stateSyncPresent && i == len(txs)-1 { + callmsg := prepareCallMessage(msg) + if _, err := statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { + failed = err + break + } + } else { + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { + failed = err + break + } } + // Finalize the state so any modifications are written to the trie // Only delete empty objects if EIP158/161 (a.k.a Spurious Dragon) is in effect statedb.Finalise(vmenv.ChainConfig().IsEIP158(block.Number())) From bfd4f3e74e8f8f2ab3fc1429ec68c9b2f6ffb85a Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Thu, 13 Oct 2022 15:55:45 +0530 Subject: [PATCH 33/70] chg : support state-sync in newcli server debugBorBlock --- eth/tracers/api_bor.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/eth/tracers/api_bor.go b/eth/tracers/api_bor.go index 6ab1a4290a..8993b9ae38 100644 --- a/eth/tracers/api_bor.go +++ b/eth/tracers/api_bor.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/bor/statefull" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -68,14 +69,14 @@ func (api *API) traceBorBlock(ctx context.Context, block *types.Block, config *T // Execute all the transaction contained within the block concurrently var ( - signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) - txs = block.Transactions() - deleteEmptyObjects = api.backend.ChainConfig().IsEIP158(block.Number()) + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + txs, stateSyncPresent = api.getAllBlockTransactions(ctx, block) + deleteEmptyObjects = api.backend.ChainConfig().IsEIP158(block.Number()) ) blockCtx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) - traceTxn := func(indx int, tx *types.Transaction) *TxTraceResult { + traceTxn := func(indx int, tx *types.Transaction, borTx bool) *TxTraceResult { message, _ := tx.AsMessage(signer, block.BaseFee()) txContext := core.NewEVMTxContext(message) @@ -88,7 +89,15 @@ func (api *API) traceBorBlock(ctx context.Context, block *types.Block, config *T // Not sure if we need to do this statedb.Prepare(tx.Hash(), indx) - execRes, err := core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) + var execRes *core.ExecutionResult + + if borTx { + callmsg := prepareCallMessage(message) + execRes, err = statefull.ApplyBorMessage(*vmenv, callmsg) + } else { + execRes, err = core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) + } + if err != nil { return &TxTraceResult{ Error: err.Error(), @@ -115,7 +124,11 @@ func (api *API) traceBorBlock(ctx context.Context, block *types.Block, config *T } for indx, tx := range txs { - res.Transactions = append(res.Transactions, traceTxn(indx, tx)) + if stateSyncPresent && indx == len(txs)-1 { + res.Transactions = append(res.Transactions, traceTxn(indx, tx, true)) + } else { + res.Transactions = append(res.Transactions, traceTxn(indx, tx, false)) + } } return res, nil From bce66b7ef5352b687d92e373f5da5fc125be1770 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 14 Oct 2022 12:53:46 +0530 Subject: [PATCH 34/70] chg : lint files --- consensus/bor/statefull/processor.go | 3 --- eth/tracers/api.go | 17 ++++++++++++----- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/consensus/bor/statefull/processor.go b/consensus/bor/statefull/processor.go index 192ea5fa38..98ed41ed92 100644 --- a/consensus/bor/statefull/processor.go +++ b/consensus/bor/statefull/processor.go @@ -90,11 +90,9 @@ func ApplyMessage( gasUsed := initialGas - gasLeft return gasUsed, nil - } func ApplyBorMessage(vmenv vm.EVM, msg Callmsg) (*core.ExecutionResult, error) { - initialGas := msg.Gas() // Apply the transaction to the current state (included in the env) @@ -117,5 +115,4 @@ func ApplyBorMessage(vmenv vm.EVM, msg Callmsg) (*core.ExecutionResult, error) { Err: err, ReturnData: ret, }, nil - } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index c6398d1a8a..baf9417b82 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -308,6 +308,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config } var res interface{} + var err error if stateSyncPresent && i == len(txs)-1 { @@ -528,9 +529,8 @@ func (api *API) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, } func prepareCallMessage(msg core.Message) statefull.Callmsg { - return statefull.Callmsg{ - ethereum.CallMsg{ + CallMsg: ethereum.CallMsg{ From: msg.From(), To: msg.To(), Gas: msg.Gas(), @@ -541,7 +541,6 @@ func prepareCallMessage(msg core.Message) statefull.Callmsg { Data: msg.Data(), AccessList: msg.AccessList(), }} - } // IntermediateRoots executes a block (bad- or canon- or side-), and returns a list @@ -577,6 +576,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) deleteEmptyObjects = chainConfig.IsEIP158(block.Number()) ) + txs, stateSyncPresent := api.getAllBlockTransactions(ctx, block) for i, tx := range txs { var ( @@ -585,6 +585,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) ) statedb.Prepare(tx.Hash(), i) + if stateSyncPresent && i == len(txs)-1 { callmsg := prepareCallMessage(msg) @@ -598,7 +599,6 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. return roots, nil } - } else { if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) @@ -677,7 +677,9 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac TxIndex: task.index, TxHash: txs[task.index].Hash(), } + var res interface{} + var err error if stateSyncPresent && task.index == len(txs)-1 { res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config, true) @@ -793,6 +795,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block canon = false } } + txs, stateSyncPresent := api.getAllBlockTransactions(ctx, block) for i, tx := range txs { // Prepare the trasaction for un-traced execution @@ -828,9 +831,11 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) statedb.Prepare(tx.Hash(), i) + if stateSyncPresent && i == len(txs)-1 { callmsg := prepareCallMessage(msg) _, err = statefull.ApplyBorMessage(*vmenv, callmsg) + if writer != nil { writer.Flush() } @@ -910,6 +915,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * TxIndex: int(index), TxHash: hash, } + return api.traceTx(ctx, msg, txctx, vmctx, statedb, config, borTx) } @@ -964,6 +970,7 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc Reexec: config.Reexec, } } + return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig, borTx) } @@ -1011,12 +1018,12 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex statedb.Prepare(txctx.TxHash, txctx.TxIndex) var result *core.ExecutionResult + if borTx { callmsg := prepareCallMessage(message) if result, err = statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } - } else { result, err = core.ApplyMessage(vmenv, message, new(core.GasPool).AddGas(message.Gas())) if err != nil { From 967b20ebf782c14c030d4e3e96fb8df89d8d7718 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 04:41:49 +0530 Subject: [PATCH 35/70] chg : major fix --- eth/tracers/api.go | 123 +++++++++++++++++++++++++++++----------- eth/tracers/api_test.go | 6 +- 2 files changed, 94 insertions(+), 35 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index baf9417b82..135f1156cd 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -191,9 +191,11 @@ func (api *API) getAllBlockTransactions(ctx context.Context, block *types.Block) // TraceConfig holds extra parameters to trace functions. type TraceConfig struct { *logger.Config - Tracer *string - Timeout *string - Reexec *uint64 + Tracer *string + Timeout *string + Reexec *uint64 + BorTraceEnabled *bool + BorTx *bool } // TraceCallConfig is the config for traceCall API. It holds one more @@ -209,8 +211,9 @@ type TraceCallConfig struct { // StdTraceConfig holds extra parameters to standard-json trace functions. type StdTraceConfig struct { logger.Config - Reexec *uint64 - TxHash common.Hash + Reexec *uint64 + TxHash common.Hash + BorTrace *bool } // txTraceResult is the result of a single transaction trace. @@ -264,6 +267,11 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf // executes all the transactions contained within. The return value will be one item // per transaction, dependent on the requested tracer. func (api *API) traceChain(ctx context.Context, start, end *types.Block, config *TraceConfig) (*rpc.Subscription, error) { + if config == nil { + config = &TraceConfig{ + BorTraceEnabled: newBoolPtr(false), + } + } // Tracing a chain is a **long** operation, only do with subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -312,9 +320,14 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config var err error if stateSyncPresent && i == len(txs)-1 { - res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config, true) + if *config.BorTraceEnabled { + config.BorTx = newBoolPtr(true) + res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) + } else { + break + } } else { - res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config, false) + res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) } if err != nil { @@ -466,6 +479,11 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config return sub, nil } +func newBoolPtr(bb bool) *bool { + b := bb + return &b +} + // TraceBlockByNumber returns the structured logs created during the execution of // EVM and returns them as a JSON object. func (api *API) TraceBlockByNumber(ctx context.Context, number rpc.BlockNumber, config *TraceConfig) ([]*txTraceResult, error) { @@ -546,6 +564,12 @@ func prepareCallMessage(msg core.Message) statefull.Callmsg { // IntermediateRoots executes a block (bad- or canon- or side-), and returns a list // of intermediate roots: the stateroot after each transaction. func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config *TraceConfig) ([]common.Hash, error) { + if config == nil { + config = &TraceConfig{ + BorTraceEnabled: newBoolPtr(false), + } + } + block, _ := api.blockByHash(ctx, hash) if block == nil { // Check in the bad blocks @@ -587,18 +611,23 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config statedb.Prepare(tx.Hash(), i) if stateSyncPresent && i == len(txs)-1 { - callmsg := prepareCallMessage(msg) + if *config.BorTraceEnabled { + callmsg := prepareCallMessage(msg) - if _, err := statefull.ApplyMessage(ctx, callmsg, statedb, block.Header(), api.backend.ChainConfig(), api.chainContext(ctx)); err != nil { - log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) - // We intentionally don't return the error here: if we do, then the RPC server will not - // return the roots. Most likely, the caller already knows that a certain transaction fails to - // be included, but still want the intermediate roots that led to that point. - // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be - // executable. - // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. - return roots, nil + if _, err := statefull.ApplyMessage(ctx, callmsg, statedb, block.Header(), api.backend.ChainConfig(), api.chainContext(ctx)); err != nil { + log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) + // We intentionally don't return the error here: if we do, then the RPC server will not + // return the roots. Most likely, the caller already knows that a certain transaction fails to + // be included, but still want the intermediate roots that led to that point. + // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be + // executable. + // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. + return roots, nil + } + } else { + break } + } else { if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) @@ -635,6 +664,13 @@ func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Has // executes all the transactions contained within. The return value will be one item // per transaction, dependent on the requestd tracer. func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { + + if config == nil { + config = &TraceConfig{ + BorTraceEnabled: newBoolPtr(false), + } + } + if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") } @@ -682,9 +718,14 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac var err error if stateSyncPresent && task.index == len(txs)-1 { - res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config, true) + if *config.BorTraceEnabled { + config.BorTx = newBoolPtr(true) + res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) + } else { + break + } } else { - res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config, false) + res, err = api.traceTx(ctx, msg, txctx, blockCtx, task.statedb, config) } if err != nil { results[task.index] = &txTraceResult{Error: err.Error()} @@ -708,9 +749,13 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) if stateSyncPresent && i == len(txs)-1 { - callmsg := prepareCallMessage(msg) - if _, err := statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { - failed = err + if *config.BorTraceEnabled { + callmsg := prepareCallMessage(msg) + if _, err := statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { + failed = err + break + } + } else { break } } else { @@ -738,6 +783,11 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac // and traces either a full block or an individual transaction. The return value will // be one filename per transaction traced. func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { + if config == nil { + config = &StdTraceConfig{ + BorTrace: newBoolPtr(false), + } + } // If we're tracing a single transaction, make sure it's present if config != nil && config.TxHash != (common.Hash{}) { if !api.containsTx(ctx, block, config.TxHash) { @@ -833,11 +883,15 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block statedb.Prepare(tx.Hash(), i) if stateSyncPresent && i == len(txs)-1 { - callmsg := prepareCallMessage(msg) - _, err = statefull.ApplyBorMessage(*vmenv, callmsg) + if *config.BorTrace { + callmsg := prepareCallMessage(msg) + _, err = statefull.ApplyBorMessage(*vmenv, callmsg) - if writer != nil { - writer.Flush() + if writer != nil { + writer.Flush() + } + } else { + break } } else { _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) @@ -879,7 +933,12 @@ func (api *API) containsTx(ctx context.Context, block *types.Block, hash common. // TraceTransaction returns the structured logs created during the execution of EVM // and returns them as a JSON object. -func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig, borTx bool) (interface{}, error) { +func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { + if config == nil { + config = &TraceConfig{ + BorTraceEnabled: newBoolPtr(false), + } + } tx, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) if tx == nil { // For BorTransaction, there will be no trace available @@ -916,14 +975,14 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * TxHash: hash, } - return api.traceTx(ctx, msg, txctx, vmctx, statedb, config, borTx) + return api.traceTx(ctx, msg, txctx, vmctx, statedb, config) } // TraceCall lets you trace a given eth_call. It collects the structured logs // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. // You can provide -2 as a block number to trace on top of the pending block. -func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig, borTx bool) (interface{}, error) { +func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( err error @@ -971,13 +1030,13 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc } } - return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig, borTx) + return api.traceTx(ctx, msg, new(Context), vmctx, statedb, traceConfig) } // traceTx configures a new tracer according to the provided configuration, and // executes the given message in the provided environment. The return value will // be tracer dependent. -func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig, borTx bool) (interface{}, error) { +func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { // Assemble the structured logger or the JavaScript tracer var ( tracer vm.EVMLogger @@ -1019,7 +1078,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex var result *core.ExecutionResult - if borTx { + if *config.BorTx { callmsg := prepareCallMessage(message) if result, err = statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { return nil, fmt.Errorf("tracing failed: %w", err) diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 32c36f16ee..aa9f913396 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -290,7 +290,7 @@ func TestTraceCall(t *testing.T) { }, } for _, testspec := range testSuite { - result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config, false) + result, err := api.TraceCall(context.Background(), testspec.call, rpc.BlockNumberOrHash{BlockNumber: &testspec.blockNumber}, testspec.config) if testspec.expectErr != nil { if err == nil { t.Errorf("Expect error %v, get nothing", testspec.expectErr) @@ -330,7 +330,7 @@ func TestTraceTransaction(t *testing.T) { b.AddTx(tx) target = tx.Hash() })) - result, err := api.TraceTransaction(context.Background(), target, nil, false) + result, err := api.TraceTransaction(context.Background(), target, nil) if err != nil { t.Errorf("Failed to trace transaction %v", err) } @@ -513,7 +513,7 @@ func TestTracingWithOverrides(t *testing.T) { }, } for i, tc := range testSuite { - result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config, false) + result, err := api.TraceCall(context.Background(), tc.call, rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, tc.config) if tc.expectErr != nil { if err == nil { t.Errorf("test %d: want error %v, have nothing", i, tc.expectErr) From 06a8f2870337dc84ea238ab616fd61c1080fa08b Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 04:56:21 +0530 Subject: [PATCH 36/70] chg : minor fix --- eth/tracers/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 135f1156cd..c5ba6b9779 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -270,6 +270,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config if config == nil { config = &TraceConfig{ BorTraceEnabled: newBoolPtr(false), + BorTx: newBoolPtr(false), } } // Tracing a chain is a **long** operation, only do with subscriptions @@ -567,6 +568,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config if config == nil { config = &TraceConfig{ BorTraceEnabled: newBoolPtr(false), + BorTx: newBoolPtr(false), } } @@ -668,6 +670,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if config == nil { config = &TraceConfig{ BorTraceEnabled: newBoolPtr(false), + BorTx: newBoolPtr(false), } } @@ -937,6 +940,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * if config == nil { config = &TraceConfig{ BorTraceEnabled: newBoolPtr(false), + BorTx: newBoolPtr(false), } } tx, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) @@ -1078,6 +1082,10 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex var result *core.ExecutionResult + if config.BorTx == nil { + config.BorTx = newBoolPtr(false) + } + if *config.BorTx { callmsg := prepareCallMessage(message) if result, err = statefull.ApplyBorMessage(*vmenv, callmsg); err != nil { From 5e574bf129aef96dc7f0f88ef7ed9021ee2a177d Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 05:38:45 +0530 Subject: [PATCH 37/70] chg : fix derference error on config without borTraceEnabled --- eth/tracers/api.go | 47 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index c5ba6b9779..4534f4675f 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -65,6 +65,8 @@ const ( defaultTracechainMemLimit = common.StorageSize(500 * 1024 * 1024) ) +var defaultBorTraceEnabled = newBoolPtr(false) + // Backend interface provides the common API services (that are provided by // both full and light clients) with access to necessary functions. type Backend interface { @@ -269,10 +271,13 @@ func (api *API) TraceChain(ctx context.Context, start, end rpc.BlockNumber, conf func (api *API) traceChain(ctx context.Context, start, end *types.Block, config *TraceConfig) (*rpc.Subscription, error) { if config == nil { config = &TraceConfig{ - BorTraceEnabled: newBoolPtr(false), + BorTraceEnabled: defaultBorTraceEnabled, BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } // Tracing a chain is a **long** operation, only do with subscriptions notifier, supported := rpc.NotifierFromContext(ctx) if !supported { @@ -308,6 +313,10 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config blockCtx := core.NewEVMBlockContext(task.block.Header(), api.chainContext(localctx), nil) // Trace all the transactions contained within txs, stateSyncPresent := api.getAllBlockTransactions(ctx, task.block) + if !*config.BorTraceEnabled && stateSyncPresent { + txs = txs[:len(txs)-1] + stateSyncPresent = false + } for i, tx := range txs { msg, _ := tx.AsMessage(signer, task.block.BaseFee()) txctx := &Context{ @@ -324,8 +333,6 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config if *config.BorTraceEnabled { config.BorTx = newBoolPtr(true) res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) - } else { - break } } else { res, err = api.traceTx(localctx, msg, txctx, blockCtx, task.statedb, config) @@ -567,10 +574,13 @@ func prepareCallMessage(msg core.Message) statefull.Callmsg { func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config *TraceConfig) ([]common.Hash, error) { if config == nil { config = &TraceConfig{ - BorTraceEnabled: newBoolPtr(false), + BorTraceEnabled: defaultBorTraceEnabled, BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } block, _ := api.blockByHash(ctx, hash) if block == nil { @@ -669,10 +679,13 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if config == nil { config = &TraceConfig{ - BorTraceEnabled: newBoolPtr(false), + BorTraceEnabled: defaultBorTraceEnabled, BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") @@ -779,7 +792,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac if failed != nil { return nil, failed } - return results, nil + + if !*config.BorTraceEnabled && stateSyncPresent { + return results[:len(results)-1], nil + } else { + return results, nil + } } // standardTraceBlockToFile configures a new tracer which uses standard JSON output, @@ -939,10 +957,14 @@ func (api *API) containsTx(ctx context.Context, block *types.Block, hash common. func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config *TraceConfig) (interface{}, error) { if config == nil { config = &TraceConfig{ - BorTraceEnabled: newBoolPtr(false), + BorTraceEnabled: defaultBorTraceEnabled, BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } + tx, blockHash, blockNumber, index, err := api.backend.GetTransaction(ctx, hash) if tx == nil { // For BorTransaction, there will be no trace available @@ -1041,6 +1063,17 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc // executes the given message in the provided environment. The return value will // be tracer dependent. func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Context, vmctx vm.BlockContext, statedb *state.StateDB, config *TraceConfig) (interface{}, error) { + + if config == nil { + config = &TraceConfig{ + BorTraceEnabled: defaultBorTraceEnabled, + BorTx: newBoolPtr(false), + } + } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } + // Assemble the structured logger or the JavaScript tracer var ( tracer vm.EVMLogger From 3a0cedc4b17ee3402cc5269452871ccb931ed5ab Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 06:26:43 +0530 Subject: [PATCH 38/70] chg : standardTraceBlockToFile fix --- eth/tracers/api.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 4534f4675f..ad52373038 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -213,9 +213,9 @@ type TraceCallConfig struct { // StdTraceConfig holds extra parameters to standard-json trace functions. type StdTraceConfig struct { logger.Config - Reexec *uint64 - TxHash common.Hash - BorTrace *bool + Reexec *uint64 + TxHash common.Hash + BorTraceEnabled *bool } // txTraceResult is the result of a single transaction trace. @@ -806,9 +806,12 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block, config *StdTraceConfig) ([]string, error) { if config == nil { config = &StdTraceConfig{ - BorTrace: newBoolPtr(false), + BorTraceEnabled: defaultBorTraceEnabled, } } + if config.BorTraceEnabled == nil { + config.BorTraceEnabled = defaultBorTraceEnabled + } // If we're tracing a single transaction, make sure it's present if config != nil && config.TxHash != (common.Hash{}) { if !api.containsTx(ctx, block, config.TxHash) { @@ -868,6 +871,11 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } txs, stateSyncPresent := api.getAllBlockTransactions(ctx, block) + if !*config.BorTraceEnabled && stateSyncPresent { + txs = txs[:len(txs)-1] + stateSyncPresent = false + } + for i, tx := range txs { // Prepare the trasaction for un-traced execution var ( @@ -904,15 +912,13 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block statedb.Prepare(tx.Hash(), i) if stateSyncPresent && i == len(txs)-1 { - if *config.BorTrace { + if *config.BorTraceEnabled { callmsg := prepareCallMessage(msg) _, err = statefull.ApplyBorMessage(*vmenv, callmsg) if writer != nil { writer.Flush() } - } else { - break } } else { _, err = core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())) From 2ec9e7b540bf94f0cb23bed54baa666f5beafd0d Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 21 Oct 2022 06:49:01 +0530 Subject: [PATCH 39/70] add : lint --- eth/tracers/api.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index ad52373038..3fce91ac9c 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -275,6 +275,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } @@ -317,6 +318,7 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config txs = txs[:len(txs)-1] stateSyncPresent = false } + for i, tx := range txs { msg, _ := tx.AsMessage(signer, task.block.BaseFee()) txctx := &Context{ @@ -578,6 +580,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } @@ -621,7 +624,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) ) statedb.Prepare(tx.Hash(), i) - + //nolint: nestif if stateSyncPresent && i == len(txs)-1 { if *config.BorTraceEnabled { callmsg := prepareCallMessage(msg) @@ -639,7 +642,6 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config } else { break } - } else { if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) @@ -683,6 +685,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } @@ -733,6 +736,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac var res interface{} var err error + if stateSyncPresent && task.index == len(txs)-1 { if *config.BorTraceEnabled { config.BorTx = newBoolPtr(true) @@ -763,7 +767,7 @@ func (api *API) traceBlock(ctx context.Context, block *types.Block, config *Trac statedb.Prepare(tx.Hash(), i) vmenv := vm.NewEVM(blockCtx, core.NewEVMTxContext(msg), statedb, api.backend.ChainConfig(), vm.Config{}) - + //nolint: nestif if stateSyncPresent && i == len(txs)-1 { if *config.BorTraceEnabled { callmsg := prepareCallMessage(msg) @@ -809,6 +813,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block BorTraceEnabled: defaultBorTraceEnabled, } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } @@ -910,7 +915,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block // Execute the transaction and flush any traces to disk vmenv := vm.NewEVM(vmctx, txContext, statedb, chainConfig, vmConf) statedb.Prepare(tx.Hash(), i) - + //nolint: nestif if stateSyncPresent && i == len(txs)-1 { if *config.BorTraceEnabled { callmsg := prepareCallMessage(msg) @@ -967,6 +972,7 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } @@ -1076,6 +1082,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex BorTx: newBoolPtr(false), } } + if config.BorTraceEnabled == nil { config.BorTraceEnabled = defaultBorTraceEnabled } From 0bc9f60756a63a4199a4bdadb3c1185e258257b6 Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 1 Nov 2022 17:00:41 -0700 Subject: [PATCH 40/70] Remove context from ApplyMessage --- eth/tracers/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 3fce91ac9c..36296e0d42 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -629,7 +629,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config if *config.BorTraceEnabled { callmsg := prepareCallMessage(msg) - if _, err := statefull.ApplyMessage(ctx, callmsg, statedb, block.Header(), api.backend.ChainConfig(), api.chainContext(ctx)); err != nil { + if _, err := statefull.ApplyMessage(callmsg, statedb, block.Header(), api.backend.ChainConfig(), api.chainContext(ctx)); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to From 8cc0b25353df448f4e09c043981e1752fa8236d5 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 4 Nov 2022 01:24:57 +0530 Subject: [PATCH 41/70] Removed vhosts for ws and replaced cors with origins (#574) * removed vhosts for ws and added replaced cors with origins * updated script --- builder/files/config.toml | 3 +-- docs/cli/server.md | 6 +++--- internal/cli/server/config.go | 8 +++++--- internal/cli/server/flags.go | 15 ++++----------- scripts/getconfig.go | 10 ++++------ 5 files changed, 17 insertions(+), 25 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index 0d01f85d71..8f70f62a13 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -78,8 +78,7 @@ syncmode = "full" # prefix = "" # host = "localhost" # api = ["web3", "net"] - # vhosts = ["*"] - # corsdomain = ["*"] + # origins = ["*"] # [jsonrpc.graphql] # enabled = false # port = 0 diff --git a/docs/cli/server.md b/docs/cli/server.md index 5fe440b5fd..d52b135fa3 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -80,6 +80,8 @@ The ```bor server``` command runs the Bor client. - ```cache.preimages```: Enable recording the SHA3/keccak preimages of trie keys +- ```cache.triesinmemory```: Number of block states (tries) to keep in memory (default = 128) + - ```txlookuplimit```: Number of recent blocks to maintain transactions index for (default = about 56 days, 0 = entire chain) ### JsonRPC Options @@ -96,9 +98,7 @@ The ```bor server``` command runs the Bor client. - ```http.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. -- ```ws.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) - -- ```ws.vhosts```: Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. +- ```ws.origins```: Origins from which to accept websockets requests - ```graphql.corsdomain```: Comma separated list of domains from which to accept cross origin requests (browser enforced) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 86b611ac78..7070afca24 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -263,6 +263,9 @@ type APIConfig struct { // Cors is the list of Cors endpoints Cors []string `hcl:"corsdomain,optional" toml:"corsdomain,optional"` + + // Origins is the list of endpoints to accept requests from (only consumed for websockets) + Origins []string `hcl:"origins,optional" toml:"origins,optional"` } type GpoConfig struct { @@ -466,8 +469,7 @@ func DefaultConfig() *Config { Prefix: "", Host: "localhost", API: []string{"net", "web3"}, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, + Origins: []string{"localhost"}, }, Graphql: &APIConfig{ Enabled: false, @@ -921,7 +923,7 @@ func (c *Config) buildNode() (*node.Config, error) { HTTPVirtualHosts: c.JsonRPC.Http.VHost, HTTPPathPrefix: c.JsonRPC.Http.Prefix, WSModules: c.JsonRPC.Ws.API, - WSOrigins: c.JsonRPC.Ws.Cors, + WSOrigins: c.JsonRPC.Ws.Origins, WSPathPrefix: c.JsonRPC.Ws.Prefix, GraphQLCors: c.JsonRPC.Graphql.Cors, GraphQLVirtualHosts: c.JsonRPC.Graphql.VHost, diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index ec74f89991..79aec3157a 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -350,17 +350,10 @@ func (c *Command) Flags() *flagset.Flagset { Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "ws.corsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced)", - Value: &c.cliConfig.JsonRPC.Ws.Cors, - Default: c.cliConfig.JsonRPC.Ws.Cors, - Group: "JsonRPC", - }) - f.SliceStringFlag(&flagset.SliceStringFlag{ - Name: "ws.vhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.", - Value: &c.cliConfig.JsonRPC.Ws.VHost, - Default: c.cliConfig.JsonRPC.Ws.VHost, + Name: "ws.origins", + Usage: "Origins from which to accept websockets requests", + Value: &c.cliConfig.JsonRPC.Ws.Origins, + Default: c.cliConfig.JsonRPC.Ws.Origins, Group: "JsonRPC", }) f.SliceStringFlag(&flagset.SliceStringFlag{ diff --git a/scripts/getconfig.go b/scripts/getconfig.go index 136b69ecab..caf3f45a8e 100644 --- a/scripts/getconfig.go +++ b/scripts/getconfig.go @@ -95,7 +95,7 @@ var flagMap = map[string][]string{ "override.arrowglacier": {"notABoolFlag", "No"}, "override.terminaltotaldifficulty": {"notABoolFlag", "No"}, "verbosity": {"notABoolFlag", "YesFV"}, - "ws.origins": {"notABoolFlag", "YesF"}, + "ws.origins": {"notABoolFlag", "No"}, } // map from cli flags to corresponding toml tags @@ -150,8 +150,7 @@ var nameTagMap = map[string]string{ "ipcpath": "ipcpath", "1-corsdomain": "http.corsdomain", "1-vhosts": "http.vhosts", - "2-corsdomain": "ws.corsdomain", - "2-vhosts": "ws.vhosts", + "origins": "ws.origins", "3-corsdomain": "graphql.corsdomain", "3-vhosts": "graphql.vhosts", "1-enabled": "http", @@ -226,9 +225,8 @@ var replacedFlagsMapFlagAndValue = map[string]map[string]map[string]string{ }, } -var replacedFlagsMapFlag = map[string]string{ - "ws.origins": "ws.corsdomain", -} +// Do not remove +var replacedFlagsMapFlag = map[string]string{} var currentBoolFlags = []string{ "snapshot", From 06c8ca661198eef3254989550a2ce8b102a17af8 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Sun, 20 Nov 2022 21:47:00 +0530 Subject: [PATCH 42/70] fix unused imports --- cmd/geth/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index d152dcbc87..80bf95b1ac 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -18,7 +18,6 @@ package main import ( "fmt" - "io/ioutil" "math/big" "os" "time" @@ -26,8 +25,6 @@ import ( "github.com/BurntSushi/toml" "gopkg.in/urfave/cli.v1" - "github.com/BurntSushi/toml" - "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" From 07f9741e8a8609f74cbea03141ba3e9a4d48d930 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 21 Nov 2022 14:53:22 +0530 Subject: [PATCH 43/70] add : bor.logs in custom GetFilterLogs --- eth/filters/api.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 2faf19aa79..5272b59480 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -416,10 +416,19 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty return nil, fmt.Errorf("filter not found") } + borConfig := api.chainConfig.Bor + var filter *Filter + var borLogsFilter *BorBlockLogsFilter + if f.crit.BlockHash != nil { // Block filter requested, construct a single-shot filter filter = NewBlockFilter(api.backend, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + + // Block bor filter + if api.borLogs { + borLogsFilter = NewBorBlockLogsFilter(api.backend, borConfig, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + } } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -432,12 +441,27 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty } // Construct the range filter filter = NewRangeFilter(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) + + if api.borLogs { + borLogsFilter = NewBorBlockLogsRangeFilter(api.backend, borConfig, begin, end, f.crit.Addresses, f.crit.Topics) + } } // Run the filter and return all the logs logs, err := filter.Logs(ctx) if err != nil { return nil, err } + + if borLogsFilter != nil { + // Run the filter and return all the logs + borBlockLogs, err := borLogsFilter.Logs(ctx) + if err != nil { + return nil, err + } + + return returnLogs(types.MergeBorLogs(logs, borBlockLogs)), err + } + return returnLogs(logs), nil } From b6d127647437710f1fddcb9b8eafe52ecda6c479 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 21 Nov 2022 15:16:28 +0530 Subject: [PATCH 44/70] fix : lint --- eth/filters/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eth/filters/api.go b/eth/filters/api.go index 5272b59480..1997b09811 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -419,6 +419,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty borConfig := api.chainConfig.Bor var filter *Filter + var borLogsFilter *BorBlockLogsFilter if f.crit.BlockHash != nil { From 4cd58f176a1e00cb78b7dd903b08a59e9a63607e Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 21 Nov 2022 15:28:34 +0530 Subject: [PATCH 45/70] fix : minor fix --- eth/filters/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/filters/api.go b/eth/filters/api.go index 1997b09811..ad24bf34dd 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -460,7 +460,7 @@ func (api *PublicFilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*ty return nil, err } - return returnLogs(types.MergeBorLogs(logs, borBlockLogs)), err + return returnLogs(types.MergeBorLogs(logs, borBlockLogs)), nil } return returnLogs(logs), nil From 3008b042524e95d9e566e4a2ac06f79a545e830d Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 21 Nov 2022 18:23:12 +0530 Subject: [PATCH 46/70] add : statesync tx added to GetBlockTransactionCount rpc --- internal/ethapi/api.go | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a71ec3bd87..4d8187121d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1615,6 +1615,25 @@ type PublicTransactionPoolAPI struct { signer types.Signer } +// returns block transactions along with state-sync transaction if present +func (api *PublicTransactionPoolAPI) getAllBlockTransactions(ctx context.Context, block *types.Block) (types.Transactions, bool) { + txs := block.Transactions() + + stateSyncPresent := false + + borReceipt := rawdb.ReadBorReceipt(api.b.ChainDb(), block.Hash(), block.NumberU64()) + if borReceipt != nil { + txHash := types.GetDerivedBorTxHash(types.BorReceiptKey(block.Number().Uint64(), block.Hash())) + if txHash != (common.Hash{}) { + borTx, _, _, _, _ := api.b.GetBorBlockTransactionWithBlockHash(ctx, txHash, block.Hash()) + txs = append(txs, borTx) + stateSyncPresent = true + } + } + + return txs, stateSyncPresent +} + // NewPublicTransactionPoolAPI creates a new RPC service with methods specific for the transaction pool. func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransactionPoolAPI { // The signer used by the API should always be the 'latest' known one because we expect @@ -1626,7 +1645,8 @@ func NewPublicTransactionPoolAPI(b Backend, nonceLock *AddrLocker) *PublicTransa // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { if block, _ := s.b.BlockByNumber(ctx, blockNr); block != nil { - n := hexutil.Uint(len(block.Transactions())) + txs, _ := s.getAllBlockTransactions(ctx, block) + n := hexutil.Uint(len(txs)) return &n } return nil @@ -1635,7 +1655,8 @@ func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByNumber(ctx context. // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. func (s *PublicTransactionPoolAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { if block, _ := s.b.BlockByHash(ctx, blockHash); block != nil { - n := hexutil.Uint(len(block.Transactions())) + txs, _ := s.getAllBlockTransactions(ctx, block) + n := hexutil.Uint(len(txs)) return &n } return nil From 5f8f8988ec405c585abbb94668c75d30727cc6ca Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 21 Nov 2022 18:28:23 +0530 Subject: [PATCH 47/70] lint : fix lint --- internal/ethapi/api.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 4d8187121d..082dfea66f 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1616,6 +1616,7 @@ type PublicTransactionPoolAPI struct { } // returns block transactions along with state-sync transaction if present +// nolint: unparam func (api *PublicTransactionPoolAPI) getAllBlockTransactions(ctx context.Context, block *types.Block) (types.Transactions, bool) { txs := block.Transactions() From ac0593d6e17d440c9bbd8e8d349b81b97cd31a38 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 22 Nov 2022 01:20:09 +0530 Subject: [PATCH 48/70] mumbai fork - 13th dewc --- builder/files/genesis-testnet-v4.json | 13 +++++++++---- internal/cli/server/chains/mumbai.go | 9 +++++++-- .../chains/test_files/chain_legacy_test.json | 15 ++++++++++----- .../cli/server/chains/test_files/chain_test.json | 15 ++++++++++----- params/config.go | 9 +++++++-- 5 files changed, 43 insertions(+), 18 deletions(-) diff --git a/builder/files/genesis-testnet-v4.json b/builder/files/genesis-testnet-v4.json index 8a0af45088..fe066411a3 100644 --- a/builder/files/genesis-testnet-v4.json +++ b/builder/files/genesis-testnet-v4.json @@ -15,19 +15,24 @@ "londonBlock": 22640000, "bor": { "jaipurBlock": 22770000, + "delhiBlock": 29638656, "period": { "0": 2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "producerDelay": { - "0": 6 + "0": 6, + "29638656": 4 }, "sprint": { - "0": 64 + "0": 64, + "29638656": 16 }, "backupMultiplier": { "0": 2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "validatorContract": "0x0000000000000000000000000000000000001000", "stateReceiverContract": "0x0000000000000000000000000000000000001001", diff --git a/internal/cli/server/chains/mumbai.go b/internal/cli/server/chains/mumbai.go index 3858ad68d6..64a5b80060 100644 --- a/internal/cli/server/chains/mumbai.go +++ b/internal/cli/server/chains/mumbai.go @@ -30,19 +30,24 @@ var mumbaiTestnet = &Chain{ LondonBlock: big.NewInt(22640000), Bor: ¶ms.BorConfig{ JaipurBlock: big.NewInt(22770000), + DelhiBlock: big.NewInt(29638656), Period: map[string]uint64{ "0": 2, "25275000": 5, + "29638656": 2, }, ProducerDelay: map[string]uint64{ - "0": 6, + "0": 6, + "29638656": 4, }, Sprint: map[string]uint64{ - "0": 64, + "0": 64, + "29638656": 16, }, BackupMultiplier: map[string]uint64{ "0": 2, "25275000": 5, + "29638656": 2, }, ValidatorContract: "0x0000000000000000000000000000000000001000", StateReceiverContract: "0x0000000000000000000000000000000000001001", diff --git a/internal/cli/server/chains/test_files/chain_legacy_test.json b/internal/cli/server/chains/test_files/chain_legacy_test.json index 7adf825b0e..b97c8b1f8e 100644 --- a/internal/cli/server/chains/test_files/chain_legacy_test.json +++ b/internal/cli/server/chains/test_files/chain_legacy_test.json @@ -17,17 +17,21 @@ "bor": { "period": { "0": 2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "producerDelay": { - "0": 6 + "0": 6, + "29638656": 4 }, "sprint": { - "0": 64 + "0": 64, + "29638656": 16 }, "backupMultiplier": { "0": 2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "validatorContract": "0x0000000000000000000000000000000000001000", "stateReceiverContract": "0x0000000000000000000000000000000000001001", @@ -43,7 +47,8 @@ "burntContract": { "22640000": "0x70bcA57F4579f58670aB2d18Ef16e02C17553C38" }, - "jaipurBlock": 22770000 + "jaipurBlock": 22770000, + "delhiBlock": 29638656 } }, "nonce": "0x0", diff --git a/internal/cli/server/chains/test_files/chain_test.json b/internal/cli/server/chains/test_files/chain_test.json index 9dc949e6f2..7907adfcfa 100644 --- a/internal/cli/server/chains/test_files/chain_test.json +++ b/internal/cli/server/chains/test_files/chain_test.json @@ -19,17 +19,21 @@ "bor":{ "period":{ "0":2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "producerDelay":{ - "0": 6 + "0": 6, + "29638656": 4 }, "sprint":{ - "0": 64 + "0": 64, + "29638656": 16 }, "backupMultiplier":{ "0": 2, - "25275000": 5 + "25275000": 5, + "29638656": 2 }, "validatorContract":"0x0000000000000000000000000000000000001000", "stateReceiverContract":"0x0000000000000000000000000000000000001001", @@ -45,7 +49,8 @@ "burntContract":{ "22640000":"0x70bcA57F4579f58670aB2d18Ef16e02C17553C38" }, - "jaipurBlock":22770000 + "jaipurBlock":22770000, + "delhiBlock": 29638656 } }, "nonce":"0x0", diff --git a/params/config.go b/params/config.go index a480218e22..d97d6957fa 100644 --- a/params/config.go +++ b/params/config.go @@ -350,19 +350,24 @@ var ( LondonBlock: big.NewInt(22640000), Bor: &BorConfig{ JaipurBlock: big.NewInt(22770000), + DelhiBlock: big.NewInt(29638656), Period: map[string]uint64{ "0": 2, "25275000": 5, + "29638656": 2, }, ProducerDelay: map[string]uint64{ - "0": 6, + "0": 6, + "29638656": 4, }, Sprint: map[string]uint64{ - "0": 64, + "0": 64, + "29638656": 16, }, BackupMultiplier: map[string]uint64{ "0": 2, "25275000": 5, + "29638656": 2, }, ValidatorContract: "0x0000000000000000000000000000000000001000", StateReceiverContract: "0x0000000000000000000000000000000000001001", From 8ca3251aab76a498a9a5135209558ae950bc99d4 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 22 Nov 2022 01:27:22 +0530 Subject: [PATCH 49/70] version change --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index abb840e986..64b58283bb 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 0 // Major version component of the current release - VersionMinor = 3 // Minor version component of the current release - VersionPatch = 1 // Patch version component of the current release - VersionMeta = "beta" // Version metadata to append to the version string + VersionMajor = 0 // Major version component of the current release + VersionMinor = 3 // Minor version component of the current release + VersionPatch = 1 // Patch version component of the current release + VersionMeta = "mumbai" // Version metadata to append to the version string ) // Version holds the textual version string. From be58978cdcb78a5c23ab9b4ac68916eb6533835b Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 22 Nov 2022 12:20:57 +0530 Subject: [PATCH 50/70] revert some changes --- cmd/utils/flags.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index a263e5ef56..5de9c982d4 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -250,13 +250,13 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } - EthRequiredBlocksFlag = cli.StringFlag{ + EthPeerRequiredBlocksFlag = cli.StringFlag{ Name: "eth.requiredblocks", Usage: "Comma separated block number-to-hash mappings to require for peering (=)", } LegacyWhitelistFlag = cli.StringFlag{ Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --peer.requiredblocks)", } BloomFilterSizeFlag = cli.Uint64Flag{ Name: "bloomfilter.size", @@ -1499,20 +1499,20 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { } } -func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { - requiredBlocks := ctx.GlobalString(EthRequiredBlocksFlag.Name) +func setPeerRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { + peerRequiredBlocks := ctx.GlobalString(EthPeerRequiredBlocksFlag.Name) - if requiredBlocks == "" { + if peerRequiredBlocks == "" { if ctx.GlobalIsSet(LegacyWhitelistFlag.Name) { log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks") - requiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) + peerRequiredBlocks = ctx.GlobalString(LegacyWhitelistFlag.Name) } else { return } } cfg.PeerRequiredBlocks = make(map[uint64]common.Hash) - for _, entry := range strings.Split(requiredBlocks, ",") { + for _, entry := range strings.Split(peerRequiredBlocks, ",") { parts := strings.Split(entry, "=") if len(parts) != 2 { Fatalf("Invalid required block entry: %s", entry) @@ -1592,7 +1592,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { setTxPool(ctx, &cfg.TxPool) setEthash(ctx, cfg) setMiner(ctx, &cfg.Miner) - setRequiredBlocks(ctx, cfg) + setPeerRequiredBlocks(ctx, cfg) setLes(ctx, cfg) if ctx.GlobalIsSet(BorLogsFlag.Name) { From d89759c3185ce59adf0d3086826ee513e45ca0be Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 22 Nov 2022 12:22:22 +0530 Subject: [PATCH 51/70] fix issues --- cmd/geth/main.go | 2 +- cmd/geth/usage.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index af565d71ae..600e929706 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -109,7 +109,7 @@ var ( utils.UltraLightFractionFlag, utils.UltraLightOnlyAnnounceFlag, utils.LightNoSyncServeFlag, - utils.EthRequiredBlocksFlag, + utils.EthPeerRequiredBlocksFlag, utils.LegacyWhitelistFlag, utils.BloomFilterSizeFlag, utils.CacheFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 28f8f84533..af5175cd63 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -56,7 +56,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.EthStatsURLFlag, utils.IdentityFlag, utils.LightKDFFlag, - utils.EthRequiredBlocksFlag, + utils.EthPeerRequiredBlocksFlag, }, }, { From 7ae5bce4d522c060ce89a74c723f8b7f6dad7c90 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Tue, 22 Nov 2022 12:28:57 +0530 Subject: [PATCH 52/70] minor refactor --- cmd/utils/flags.go | 6 +++--- eth/ethconfig/config.go | 2 +- eth/ethconfig/gen_config.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5de9c982d4..81ce27ef4c 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1515,15 +1515,15 @@ func setPeerRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) { for _, entry := range strings.Split(peerRequiredBlocks, ",") { parts := strings.Split(entry, "=") if len(parts) != 2 { - Fatalf("Invalid required block entry: %s", entry) + Fatalf("Invalid peer required block entry: %s", entry) } number, err := strconv.ParseUint(parts[0], 0, 64) if err != nil { - Fatalf("Invalid required block number %s: %v", parts[0], err) + Fatalf("Invalid peer required block number %s: %v", parts[0], err) } var hash common.Hash if err = hash.UnmarshalText([]byte(parts[1])); err != nil { - Fatalf("Invalid required block hash %s: %v", parts[1], err) + Fatalf("Invalid peer required block hash %s: %v", parts[1], err) } cfg.PeerRequiredBlocks[number] = hash } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8089794948..c9272758ab 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -144,7 +144,7 @@ type Config struct { TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. - // RequiredBlocks is a set of block number -> hash mappings which must be in the + // PeerRequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the // presence of these blocks for every new peer connection. PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index fdb6fe11e1..874e30dffd 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -26,7 +26,7 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` - PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` + PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ int `toml:",omitempty"` LightIngress int `toml:",omitempty"` LightEgress int `toml:",omitempty"` @@ -120,7 +120,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` - PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` + PeerRequiredBlocks map[uint64]common.Hash `toml:"-"` LightServ *int `toml:",omitempty"` LightIngress *int `toml:",omitempty"` LightEgress *int `toml:",omitempty"` From be0a2ad79f7c7e4b45a984c7bf4a302628bb72ac Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 11:24:56 +0530 Subject: [PATCH 53/70] CI: test launch devnet without hardcoded sleep time (pos-534) --- .github/workflows/ci.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bb62a8bb..0497515e3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: - "master" - "qa" - "develop" + - "raneet10/pos-534" # RMV! pull_request: branches: - "**" @@ -142,11 +143,9 @@ jobs: bash docker-heimdall-start-all.sh bash docker-bor-setup.sh bash docker-bor-start-all.sh - sleep 120 && bash ganache-deployment-bor.sh - sleep 120 && bash ganache-deployment-sync.sh - sleep 120 - docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'admin.peers'" - docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'eth.blockNumber'" + timeout 2m bash bor/integration-tests/bor_health.sh + bash ganache-deployment-bor.sh + bash ganache-deployment-sync.sh - name: Run smoke tests run: | From c686660fe5de59a6265b641d7c902e7e18de7ec0 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 14:05:58 +0530 Subject: [PATCH 54/70] CI: try using checked out bor path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5f4614bc31..c611b51d4a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,7 @@ jobs: bash docker-heimdall-start-all.sh bash docker-bor-setup.sh bash docker-bor-start-all.sh - cd code/ + cd - timeout 2m bash bor/integration-tests/bor_health.sh cd - bash ganache-deployment-bor.sh From 5df25bd360c8f9ad1945d9e1db868febed30445e Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 14:50:18 +0530 Subject: [PATCH 55/70] CI: fix missing ; --- integration-tests/bor_health.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bor_health.sh b/integration-tests/bor_health.sh index 8e79197d00..020aa25063 100644 --- a/integration-tests/bor_health.sh +++ b/integration-tests/bor_health.sh @@ -6,7 +6,7 @@ do peers=$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'admin.peers'") block-$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'eth.blockNumber'") - if [[ -n "$peers" ]] && [[ -n "$block" ]] then + if [[ -n "$peers" ]] && [[ -n "$block" ]]; then break fi done \ No newline at end of file From e378b5febc6a9388643dc1cd79bf30b0af071665 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 15:29:25 +0530 Subject: [PATCH 56/70] CI: fix assignment operator --- integration-tests/bor_health.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bor_health.sh b/integration-tests/bor_health.sh index 020aa25063..0be1d8257f 100644 --- a/integration-tests/bor_health.sh +++ b/integration-tests/bor_health.sh @@ -4,7 +4,7 @@ set -e while true do peers=$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'admin.peers'") - block-$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'eth.blockNumber'") + block=$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'eth.blockNumber'") if [[ -n "$peers" ]] && [[ -n "$block" ]]; then break From 7cb597886a1ad5239ffcba1558267580ce3c6090 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 16:20:43 +0530 Subject: [PATCH 57/70] CI: echo peers and block no. --- integration-tests/bor_health.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/integration-tests/bor_health.sh b/integration-tests/bor_health.sh index 0be1d8257f..a4ddb540cf 100644 --- a/integration-tests/bor_health.sh +++ b/integration-tests/bor_health.sh @@ -9,4 +9,7 @@ do if [[ -n "$peers" ]] && [[ -n "$block" ]]; then break fi -done \ No newline at end of file +done + +echo $peers +echo $block \ No newline at end of file From 9155cfc68bba6e779408f1315c611350d133fcaa Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 17:37:31 +0530 Subject: [PATCH 58/70] CI: cleanup --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c611b51d4a..e9e454c549 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ on: - "master" - "qa" - "develop" - - "raneet10/pos-534" # RMV! pull_request: branches: - "**" From 806ed051be09056f52e6d433585987ef112e1b40 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 25 Nov 2022 17:50:58 +0530 Subject: [PATCH 59/70] minor chg: add new line --- integration-tests/bor_health.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/bor_health.sh b/integration-tests/bor_health.sh index a4ddb540cf..3288739f85 100644 --- a/integration-tests/bor_health.sh +++ b/integration-tests/bor_health.sh @@ -12,4 +12,4 @@ do done echo $peers -echo $block \ No newline at end of file +echo $block From 50a778207e95775ea38c5ead933fec5d946e2716 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Tue, 6 Dec 2022 10:53:55 +0100 Subject: [PATCH 60/70] dev: add: pos-944: snyk and govuln integration (#578) * dev: add: pos-944 security ci and readme * dev: add: pos-944 remove linters as this is included already in build ci * dev: chg: pos-947 dependencies upgrade to solve snyk security issues * dev: chg: update security-ci * dev: chg: remove linter to allow replacements for security issues * dev: add: pos-944 verify path when updating metrics from config * dev: add: pos-944 fix linter * dev: add: pos-944 add .snyk policy file / fix snyk code vulnerabilities * dev: fix: pos-944 import common package / gitignore snyk dccache file * dev: fix: pos-944 verify canonical path for crashers * dev: fix: pos-944 linter * dev: add: pos-976 add govuln check * dev: add: pos-976 test upload with permissions * dev: add: pos-976 remove duplicated upload * dev: add: pos-976 report upload * dev: add: pos-976 remove upload * dev: fix: pos-944 fix govuln action * dev: fix: pos-944 move govulncheck to security-ci * dev: fix: pos-944 bump golvun action and golang versions * dev: fix: pos-944 remove persmissions and fix conflicts * dev: chg: restore err msg * dev: chg: remove duplicated function * dev: chg: sort import * dev: chg: fix linter * dev: add: use common VerifyCrasher function to avoid duplications / replace deprecated ioutils.ReadFile * dev: fix: typo --- .github/workflows/security-ci.yml | 64 +++++++++ .gitignore | 2 + .golangci.yml | 10 +- .snyk | 37 +++++ SECURITY.md | 181 ++----------------------- build/ci.go | 38 +++--- cmd/faucet/faucet.go | 11 +- common/path.go | 29 ++++ go.mod | 4 +- go.sum | 7 +- metrics/metrics.go | 11 +- rlp/rlpgen/main.go | 13 +- scripts/getconfig.go | 11 +- tests/fuzzers/difficulty/debug/main.go | 11 +- tests/fuzzers/les/debug/main.go | 11 +- tests/fuzzers/rangeproof/debug/main.go | 11 +- tests/fuzzers/snap/debug/main.go | 11 +- tests/fuzzers/stacktrie/debug/main.go | 11 +- tests/fuzzers/vflux/debug/main.go | 11 +- 19 files changed, 251 insertions(+), 233 deletions(-) create mode 100644 .github/workflows/security-ci.yml create mode 100644 .snyk diff --git a/.github/workflows/security-ci.yml b/.github/workflows/security-ci.yml new file mode 100644 index 0000000000..5dc2b221db --- /dev/null +++ b/.github/workflows/security-ci.yml @@ -0,0 +1,64 @@ +name: Security CI +on: [push, pull_request] + +jobs: + snyk: + name: Snyk and Publish + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@master + - name: Run Snyk to check for vulnerabilities + uses: snyk/actions/golang@master + continue-on-error: true + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --org=${{ secrets.SNYK_ORG }} --severity-threshold=medium --sarif-file-output=snyk.sarif + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: snyk.sarif + + snyk-code: + name: Snyk Code and Publish + runs-on: ubuntu-latest + continue-on-error: true + steps: + - name: Checkout Source + uses: actions/checkout@master + - name: Run Snyk SAST to check for code vulnerabilities + uses: snyk/actions/golang@master + continue-on-error: true + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + args: --org=${{ secrets.SNYK_ORG }} --sarif-file-output=snyk.sarif + command: code test + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: snyk.sarif + + govuln: + name: Run govuln check and Publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Running govulncheck + uses: Templum/govulncheck-action@v0.0.8 + continue-on-error: true + env: + DEBUG: "true" + with: + go-version: 1.19 + vulncheck-version: latest + package: ./... + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-vuln: true + + - name: Upload govulncheck report + uses: actions/upload-artifact@v3 + with: + name: raw-report + path: raw-report.json diff --git a/.gitignore b/.gitignore index cd3c25a6a8..0d2f13decf 100644 --- a/.gitignore +++ b/.gitignore @@ -54,4 +54,6 @@ profile.cov dist +.dccache + *.csv diff --git a/.golangci.yml b/.golangci.yml index 89eebfe9fe..33ddb3bae1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -30,7 +30,7 @@ linters: - gocognit - gofmt # - gomnd - - gomoddirectives + # - gomoddirectives - gosec - makezero - nestif @@ -65,10 +65,10 @@ linters-settings: goimports: local-prefixes: github.com/ethereum/go-ethereum - + nestif: min-complexity: 5 - + prealloc: for-loops: true @@ -79,7 +79,7 @@ linters-settings: # By default list of stable checks is used. enabled-checks: - badLock - - filepathJoin + - filepathJoin - sortSlice - sprintfQuotedString - syncMapLoadAndDelete @@ -185,4 +185,4 @@ issues: max-issues-per-linter: 0 max-same-issues: 0 #new: true - new-from-rev: origin/master \ No newline at end of file + new-from-rev: origin/master diff --git a/.snyk b/.snyk new file mode 100644 index 0000000000..2fa83cf27c --- /dev/null +++ b/.snyk @@ -0,0 +1,37 @@ +# Snyk (https://snyk.io) policy file, patches or ignores known vulnerabilities. +version: v1.25.0 +# ignores vulnerabilities until expiry date; change duration by modifying expiry date +ignore: + 'snyk:lic:golang:github.com:karalabe:usb:LGPL-3.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:06:37.028Z + 'snyk:lic:golang:github.com:mitchellh:cli:MPL-2.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:07:42.661Z + 'snyk:lic:golang:github.com:hashicorp:hcl:v2:MPL-2.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:08.112Z + 'snyk:lic:golang:github.com:hashicorp:go-multierror:MPL-2.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:14.673Z + 'snyk:lic:golang:github.com:hashicorp:go-bexpr:MPL-2.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:21.843Z + 'snyk:lic:golang:github.com:hashicorp:errwrap:MPL-2.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:28.257Z + 'snyk:lic:golang:github.com:ethereum:go-ethereum:LGPL-3.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:35.273Z + 'snyk:lic:golang:github.com:maticnetwork:polyproto:GPL-3.0': + - '*': + reason: 'As open source org, we have no issues with licenses' + created: 2022-11-11T08:09:41.635Z +patch: {} diff --git a/SECURITY.md b/SECURITY.md index 41b900d5e9..d082838a32 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,175 +1,14 @@ -# Security Policy +# Polygon Technology Security Information -## Supported Versions +## Link to vulnerability disclosure details (Bug Bounty) +- Websites and Applications: https://hackerone.com/polygon-technology +- Smart Contracts: https://immunefi.com/bounty/polygon -Please see [Releases](https://github.com/ethereum/go-ethereum/releases). We recommend using the [most recently released version](https://github.com/ethereum/go-ethereum/releases/latest). +## Languages that our team speaks and understands. +Preferred-Languages: en -## Audit reports +## Security-related job openings at Polygon. +https://polygon.technology/careers -Audit reports are published in the `docs` folder: https://github.com/ethereum/go-ethereum/tree/master/docs/audits - -| Scope | Date | Report Link | -| ------- | ------- | ----------- | -| `geth` | 20170425 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2017-04-25_Geth-audit_Truesec.pdf) | -| `clef` | 20180914 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2018-09-14_Clef-audit_NCC.pdf) | -| `Discv5` | 20191015 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2019-10-15_Discv5_audit_LeastAuthority.pdf) | -| `Discv5` | 20200124 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2020-01-24_DiscV5_audit_Cure53.pdf) | - -## Reporting a Vulnerability - -**Please do not file a public ticket** mentioning the vulnerability. - -To find out how to disclose a vulnerability in Ethereum visit [https://bounty.ethereum.org](https://bounty.ethereum.org) or email bounty@ethereum.org. Please read the [disclosure page](https://github.com/ethereum/go-ethereum/security/advisories?state=published) for more information about publicly disclosed security vulnerabilities. - -Use the built-in `geth version-check` feature to check whether the software is affected by any known vulnerability. This command will fetch the latest [`vulnerabilities.json`](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities.json) file which contains known security vulnerabilities concerning `geth`, and cross-check the data against its own version number. - -The following key may be used to communicate sensitive information to developers. - -Fingerprint: `AE96 ED96 9E47 9B00 84F3 E17F E88D 3334 FA5F 6A0A` - -``` ------BEGIN PGP PUBLIC KEY BLOCK----- -Version: SKS 1.1.6 -Comment: Hostname: pgp.mit.edu - -mQINBFgl3tgBEAC8A1tUBkD9YV+eLrOmtgy+/JS/H9RoZvkg3K1WZ8IYfj6iIRaYneAk3Bp1 -82GUPVz/zhKr2g0tMXIScDR3EnaDsY+Qg+JqQl8NOG+Cikr1nnkG2on9L8c8yiqry1ZTCmYM -qCa2acTFqnyuXJ482aZNtB4QG2BpzfhW4k8YThpegk/EoRUim+y7buJDtoNf7YILlhDQXN8q -lHB02DWOVUihph9tUIFsPK6BvTr9SIr/eG6j6k0bfUo9pexOn7LS4SojoJmsm/5dp6AoKlac -48cZU5zwR9AYcq/nvkrfmf2WkObg/xRdEvKZzn05jRopmAIwmoC3CiLmqCHPmT5a29vEob/y -PFE335k+ujjZCPOu7OwjzDk7M0zMSfnNfDq8bXh16nn+ueBxJ0NzgD1oC6c2PhM+XRQCXCho -yI8vbfp4dGvCvYqvQAE1bWjqnumZ/7vUPgZN6gDfiAzG2mUxC2SeFBhacgzDvtQls+uuvm+F -nQOUgg2Hh8x2zgoZ7kqV29wjaUPFREuew7e+Th5BxielnzOfVycVXeSuvvIn6cd3g/s8mX1c -2kLSXJR7+KdWDrIrR5Az0kwAqFZt6B6QTlDrPswu3mxsm5TzMbny0PsbL/HBM+GZEZCjMXxB -8bqV2eSaktjnSlUNX1VXxyOxXA+ZG2jwpr51egi57riVRXokrQARAQABtDRFdGhlcmV1bSBG -b3VuZGF0aW9uIEJ1ZyBCb3VudHkgPGJvdW50eUBldGhlcmV1bS5vcmc+iQIcBBEBCAAGBQJa -FCY6AAoJEHoMA3Q0/nfveH8P+gJBPo9BXZL8isUfbUWjwLi81Yi70hZqIJUnz64SWTqBzg5b -mCZ69Ji5637THsxQetS2ARabz0DybQ779FhD/IWnqV9T3KuBM/9RzJtuhLzKCyMrAINPMo28 -rKWdunHHarpuR4m3tL2zWJkle5QVYb+vkZXJJE98PJw+N4IYeKKeCs2ubeqZu636GA0sMzzB -Jn3m/dRRA2va+/zzbr6F6b51ynzbMxWKTsJnstjC8gs8EeI+Zcd6otSyelLtCUkk3h5sTvpV -Wv67BNSU0BYsMkxyFi9PUyy07Wixgeas89K5jG1oOtDva/FkpRHrTE/WA5OXDRcLrHJM+SwD -CwqcLQqJd09NxwUW1iKeBmPptTiOGu1Gv2o7aEyoaWrHRBO7JuYrQrj6q2B3H1Je0zjAd2qt -09ni2bLwLn4LA+VDpprNTO+eZDprv09s2oFSU6NwziHybovu0y7X4pADGkK2evOM7c86PohX -QRQ1M1T16xLj6wP8/Ykwl6v/LUk7iDPXP3GPILnh4YOkwBR3DsCOPn8098xy7FxEELmupRzt -Cj9oC7YAoweeShgUjBPzb+nGY1m6OcFfbUPBgFyMMfwF6joHbiVIO+39+Ut2g2ysZa7KF+yp -XqVDqyEkYXsOLb25OC7brt8IJEPgBPwcHK5GNag6RfLxnQV+iVZ9KNH1yQgSiQI+BBMBAgAo -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAUCWglh+gUJBaNgWAAKCRDojTM0+l9qCgQ2 -D/4udJpV4zGIZW1yNaVvtd3vfKsTLi7GIRJLUBqVb2Yx/uhnN8jTl/tAhCVosCQ1pzvi9kMl -s8qO1vu2kw5EWFFkwK96roI8pTql3VIjwhRVQrCkR7oAk/eUd1U/nt2q6J4UTYeVgqbq4dsI -ZZTRyPJMD667YpuAIcaah+w9j/E5xksYQdMeprnDrQkkBCb4FIMqfDzBPKvEa8DcQr949K85 -kxhr6LDq9i5l4Egxt2JdH8DaR4GLca6+oHy0MyPs/bZOsfmZUObfM2oZgPpqYM96JanhzO1j -dpnItyBii2pc+kNx5nMOf4eikE/MBv+WUJ0TttWzApGGmFUzDhtuEvRH9NBjtJ/pMrYspIGu -O/QNY5KKOKQTvVIlwGcm8dTsSkqtBDSUwZyWbfKfKOI1/RhM9dC3gj5/BOY57DYYV4rdTK01 -ZtYjuhdfs2bhuP1uF/cgnSSZlv8azvf7Egh7tHPnYxvLjfq1bJAhCIX0hNg0a81/ndPAEFky -fSko+JPKvdSvsUcSi2QQ4U2HX//jNBjXRfG4F0utgbJnhXzEckz6gqt7wSDZH2oddVuO8Ssc -T7sK+CdXthSKnRyuI+sGUpG+6glpKWIfYkWFKNZWuQ+YUatY3QEDHXTIioycSmV8p4d/g/0S -V6TegidLxY8bXMkbqz+3n6FArRffv5MH7qt3cYkCPgQTAQIAKAUCWCXhOwIbAwUJAeEzgAYL -CQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQ6I0zNPpfagrN/w/+Igp3vtYdNunikw3yHnYf -Jkm0MmaMDUM9mtsaXVN6xb9n25N3Xa3GWCpmdsbYZ8334tI/oQ4/NHq/bEI5WFH5F1aFkMkm -5AJVLuUkipCtmCZ5NkbRPJA9l0uNUUE6uuFXBhf4ddu7jb0jMetRF/kifJHVCCo5fISUNhLp -7bwcWq9qgDQNZNYMOo4s9WX5Tl+5x4gTZdd2/cAYt49h/wnkw+huM+Jm0GojpLqIQ1jZiffm -otf5rF4L+JhIIdW0W4IIh1v9BhHVllXw+z9oj0PALstT5h8/DuKoIiirFJ4DejU85GR1KKAS -DeO19G/lSpWj1rSgFv2N2gAOxq0X+BbQTua2jdcY6JpHR4H1JJ2wzfHsHPgDQcgY1rGlmjVF -aqU73WV4/hzXc/HshK/k4Zd8uD4zypv6rFsZ3UemK0aL2zXLVpV8SPWQ61nS03x675SmDlYr -A80ENfdqvsn00JQuBVIv4Tv0Ub7NfDraDGJCst8rObjBT/0vnBWTBCebb2EsnS2iStIFkWdz -/WXs4L4Yzre1iJwqRjiuqahZR5jHsjAUf2a0O29HVHE7zlFtCFmLPClml2lGQfQOpm5klGZF -rmvus+qZ9rt35UgWHPZezykkwtWrFOwspwuCWaPDto6tgbRJZ4ftitpdYYM3dKW9IGJXBwrt -BQrMsu+lp0vDF+yJAlUEEwEIAD8CGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAFiEErpbt -lp5HmwCE8+F/6I0zNPpfagoFAmEAEJwFCQycmLgACgkQ6I0zNPpfagpWoBAAhOcbMAUw6Zt0 -GYzT3sR5/c0iatezPzXEXJf9ebzR8M5uPElXcxcnMx1dvXZmGPXPJKCPa99WCu1NZYy8F+Wj -GTOY9tfIkvSxhys1p/giPAmvid6uQmD+bz7ivktnyzCkDWfMA+l8lsCSEqVlaq6y5T+a6SWB -6TzC2S0MPb/RrC/7DpwyrNYWumvyVJh09adm1Mw/UGgst/sZ8eMaRYEd3X0yyT1CBpX4zp2E -qQj9IEOTizvzv1x2jkHe5ZUeU3+nTBNlhSA+WFHUi0pfBdo2qog3Mv2EC1P2qMKoSdD5tPbA -zql1yKoHHnXOMsqdftGwbiv2sYXWvrYvmaCd3Ys/viOyt3HOy9uV2ZEtBd9Yqo9x/NZj8QMA -nY5k8jjrIXbUC89MqrJsQ6xxWQIg5ikMT7DvY0Ln89ev4oJyVvwIQAwCm4jUzFNm9bZLYDOP -5lGJCV7tF5NYVU7NxNM8vescKc40mVNK/pygS5mxhK9QYOUjZsIv8gddrl1TkqrFMuxFnTyN -WvzE29wFu/n4N1DkF+ZBqS70SlRvB+Hjz5LrDgEzF1Wf1eA/wq1dZbvMjjDVIc2VGlYp8Cp2 -8ob23c1seTtYXTNYgSR5go4EpH+xi+bIWv01bQQ9xGwBbT5sm4WUeWOcmX4QewzLZ3T/wK9+ -N4Ye/hmU9O34FwWJOY58EIe0OUV0aGVyZXVtIEZvdW5kYXRpb24gU2VjdXJpdHkgVGVhbSA8 -c2VjdXJpdHlAZXRoZXJldW0ub3JnPokCHAQRAQgABgUCWhQmOgAKCRB6DAN0NP5372LSEACT -wZk1TASWZj5QF7rmkIM1GEyBxLE+PundNcMgM9Ktj1315ED8SmiukNI4knVS1MY99OIgXhQl -D1foF2GKdTomrwwC4012zTNyUYCY60LnPZ6Z511HG+rZgZtZrbkz0IiUpwAlhGQND77lBqem -J3K+CFX2XpDA/ojui/kqrY4cwMT5P8xPJkwgpRgw/jgdcZyJTsXdHblV9IGU4H1Vd1SgcfAf -Db3YxDUlBtzlp0NkZqxen8irLIXUQvsfuIfRUbUSkWoK/n3U/gOCajAe8ZNF07iX4OWjH4Sw -NDA841WhFWcGE+d8+pfMVfPASU3UPKH72uw86b2VgR46Av6voyMFd1pj+yCA+YAhJuOpV4yL -QaGg2Z0kVOjuNWK/kBzp1F58DWGh4YBatbhE/UyQOqAAtR7lNf0M3QF9AdrHTxX8oZeqVW3V -Fmi2mk0NwCIUv8SSrZr1dTchp04OtyXe5gZBXSfzncCSRQIUDC8OgNWaOzAaUmK299v4bvye -uSCxOysxC7Q1hZtjzFPKdljS81mRlYeUL4fHlJU9R57bg8mriSXLmn7eKrSEDm/EG5T8nRx7 -TgX2MqJs8sWFxD2+bboVEu75yuFmZ//nmCBApAit9Hr2/sCshGIEpa9MQ6xJCYUxyqeJH+Cc -Aja0UfXhnK2uvPClpJLIl4RE3gm4OXeE1IkCPgQTAQIAKAIbAwYLCQgHAwIGFQgCCQoLBBYC -AwECHgECF4AFAloJYfoFCQWjYFgACgkQ6I0zNPpfagr4MQ//cfp3GSbSG8dkqgctW67Fy7cQ -diiTmx3cwxY+tlI3yrNmdjtrIQMzGdqtY6LNz7aN87F8mXNf+DyVHX9+wd1Y8U+E+hVCTzKC -sefUfxTz6unD9TTcGqaoelgIPMn4IiKz1RZE6eKpfDWe6q78W1Y6x1bE0qGNSjqT/QSxpezF -E/OAm/t8RRxVxDtqz8LfH2zLea5zaC+ADj8EqgY9vX9TQa4DyVV8MgOyECCCadJQCD5O5hIA -B2gVDWwrAUw+KBwskXZ7Iq4reJTKLEmt5z9zgtJ/fABwaCFt66ojwg0/RjbO9cNA3ZwHLGwU -C6hkb6bRzIoZoMfYxVS84opiqf/Teq+t/XkBYCxbSXTJDA5MKjcVuw3N6YKWbkGP/EfQThe7 -BfAKFwwIw5YmsWjHK8IQj6R6hBxzTz9rz8y1Lu8EAAFfA7OJKaboI2qbOlauH98OuOUmVtr1 -TczHO+pTcgWVN0ytq2/pX5KBf4vbmULNbg3HFRq+gHx8CW+jyXGkcqjbgU/5FwtDxeqRTdGJ -SyBGNBEU6pBNolyynyaKaaJjJ/biY27pvjymL5rlz95BH3Dn16Z4RRmqwlT6eq/wFYginujg -CCE1icqOSE+Vjl7V8tV8AcgANkXKdbBE+Q8wlKsGI/kS1w4XFAYcaNHFT8qNeS8TSFXFhvU8 -HylYxO79t56JAj4EEwECACgFAlgl3tgCGwMFCQHhM4AGCwkIBwMCBhUIAgkKCwQWAgMBAh4B -AheAAAoJEOiNMzT6X2oKmUMP/0hnaL6bVyepAq2LIdvIUbHfagt/Oo/KVfZs4bkM+xJOitJR -0kwZV9PTihXFdzhL/YNWc2+LtEBtKItqkJZKmWC0E6OPXGVuU6hfFPebuzVccYJfm0Q3Ej19 -VJI9Uomf59Bpak8HYyEED7WVQjoYn7XVPsonwus/9+LDX+c5vutbrUdbjga3KjHbewD93X4O -wVVoXyHEmU2Plyg8qvzFbNDylCWO7N2McO6SN6+7DitGZGr2+jO+P2R4RT1cnl2V3IRVcWZ0 -OTspPSnRGVr2fFiHN/+v8G/wHPLQcJZFvYPfUGNdcYbTmhWdiY0bEYXFiNrgzCCsyad7eKUR -WN9QmxqmyqLDjUEDJCAh19ES6Vg3tqGwXk+uNUCoF30ga0TxQt6UXZJDEQFAGeASQ/RqE/q1 -EAuLv8IGM8o7IqKO2pWfLuqsY6dTbKBwDzz9YOJt7EOGuPPQbHxaYStTushZmJnm7hi8lhVG -jT7qsEJdE95Il+I/mHWnXsCevaXjZugBiyV9yvOq4Hwwe2s1zKfrnQ4u0cadvGAh2eIqum7M -Y3o6nD47aJ3YmEPX/WnhI56bACa2GmWvUwjI4c0/er3esSPYnuHnM9L8Am4qQwMVSmyU80tC -MI7A9e13Mvv+RRkYFLJ7PVPdNpbW5jqX1doklFpKf6/XM+B+ngYneU+zgCUBiQJVBBMBCAA/ -AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgBYhBK6W7ZaeR5sAhPPhf+iNMzT6X2oKBQJh -ABCQBQkMnJi4AAoJEOiNMzT6X2oKAv0P+gJ3twBp5efNWyVLcIg4h4cOo9uD0NPvz8/fm2gX -FoOJL3MeigtPuSVfE9kuTaTuRbArzuFtdvH6G/kcRQvOlO4zyiIRHCk1gDHoIvvtn6RbRhVm -/Xo4uGIsFHst7n4A7BjicwEK5Op6Ih5Hoq19xz83YSBgBVk2fYEJIRyJiKFbyPjH0eSYe8v+ -Ra5/F85ugLx1P6mMVkW+WPzULns89riW7BGTnZmXFHZp8nO2pkUlcI7F3KRG7l4kmlC50ox6 -DiG/6AJCVulbAClky9C68TmJ/R1RazQxU/9IqVywsydq66tbJQbm5Z7GEti0C5jjbSRJL2oT -1xC7Rilr85PMREkPL3vegJdgj5PKlffZ/MocD/0EohiQ7wFpejFD4iTljeh0exRUwCRb6655 -9ib34JSQgU8Hl4JJu+mEgd9v0ZHD0/1mMD6fnAR84zca+O3cdASbnQmzTOKcGzLIrkE8TEnU -+2UZ8Ol7SAAqmBgzY1gKOilUho6dkyCAwNL+QDpvrITDPLEFPsjyB/M2KudZSVEn+Rletju1 -qkMW31qFMNlsbwzMZw+0USeGcs31Cs0B2/WQsro99CExlhS9auUFkmoVjJmYVTIYOM0zuPa4 -OyGspqPhRu5hEsmMDPDWD7Aad5k4GTqogQNnuKyRliZjXXrDZqFD5nfsJSL8Ky/sJGEMuQIN -BFgl3tgBEACbgq6HTN5gEBi0lkD/MafInmNi+59U5gRGYqk46WlfRjhHudXjDpgD0lolGb4h -YontkMaKRlCg2Rvgjvk3Zve0PKWjKw7gr8YBa9fMFY8BhAXI32OdyI9rFhxEZFfWAfwKVmT1 -9BdeAQRFvcfd+8w8f1XVc+zddULMJFBTr+xKDlIRWwTkdLPQeWbjo0eHl/g4tuLiLrTxVbnj -26bf+2+1DbM/w5VavzPrkviHqvKe/QP/gay4QDViWvFgLb90idfAHIdsPgflp0VDS5rVHFL6 -D73rSRdIRo3I8c8mYoNjSR4XDuvgOkAKW9LR3pvouFHHjp6Fr0GesRbrbb2EG66iPsR99MQ7 -FqIL9VMHPm2mtR+XvbnKkH2rYyEqaMbSdk29jGapkAWle4sIhSKk749A4tGkHl08KZ2N9o6G -rfUehP/V2eJLaph2DioFL1HxRryrKy80QQKLMJRekxigq8greW8xB4zuf9Mkuou+RHNmo8Pe -bHjFstLigiD6/zP2e+4tUmrT0/JTGOShoGMl8Rt0VRxdPImKun+4LOXbfOxArOSkY6i35+gs -gkkSy1gTJE0BY3S9auT6+YrglY/TWPQ9IJxWVOKlT+3WIp5wJu2bBKQ420VLqDYzkoWytel/ -bM1ACUtipMiIVeUs2uFiRjpzA1Wy0QHKPTdSuGlJPRrfcQARAQABiQIlBBgBAgAPAhsMBQJa -CWIIBQkFo2BYAAoJEOiNMzT6X2oKgSwQAKKs7BGF8TyZeIEO2EUK7R2bdQDCdSGZY06tqLFg -3IHMGxDMb/7FVoa2AEsFgv6xpoebxBB5zkhUk7lslgxvKiSLYjxfNjTBltfiFJ+eQnf+OTs8 -KeR51lLa66rvIH2qUzkNDCCTF45H4wIDpV05AXhBjKYkrDCrtey1rQyFp5fxI+0IQ1UKKXvz -ZK4GdxhxDbOUSd38MYy93nqcmclGSGK/gF8XiyuVjeifDCM6+T1NQTX0K9lneidcqtBDvlgg -JTLJtQPO33o5EHzXSiud+dKth1uUhZOFEaYRZoye1YE3yB0TNOOE8fXlvu8iuIAMBSDL9ep6 -sEIaXYwoD60I2gHdWD0lkP0DOjGQpi4ouXM3Edsd5MTi0MDRNTij431kn8T/D0LCgmoUmYYM -BgbwFhXr67axPZlKjrqR0z3F/Elv0ZPPcVg1tNznsALYQ9Ovl6b5M3cJ5GapbbvNWC7yEE1q -Scl9HiMxjt/H6aPastH63/7wcN0TslW+zRBy05VNJvpWGStQXcngsSUeJtI1Gd992YNjUJq4 -/Lih6Z1TlwcFVap+cTcDptoUvXYGg/9mRNNPZwErSfIJ0Ibnx9wPVuRN6NiCLOt2mtKp2F1p -M6AOQPpZ85vEh6I8i6OaO0w/Z0UHBwvpY6jDUliaROsWUQsqz78Z34CVj4cy6vPW2EF4iQIl -BBgBAgAPBQJYJd7YAhsMBQkB4TOAAAoJEOiNMzT6X2oKTjgP/1ojCVyGyvHMLUgnX0zwrR5Q -1M5RKFz6kHwKjODVLR3Isp8I935oTQt3DY7yFDI4t0GqbYRQMtxcNEb7maianhK2trCXfhPs -6/L04igjDf5iTcmzamXN6xnh5xkz06hZJJCMuu4MvKxC9MQHCVKAwjswl/9H9JqIBXAY3E2l -LpX5P+5jDZuPxS86p3+k4Rrdp9KTGXjiuEleM3zGlz5BLWydqovOck7C2aKh27ETFpDYY0z3 -yQ5AsPJyk1rAr0wrH6+ywmwWlzuQewavnrLnJ2M8iMFXpIhyHeEIU/f7o8f+dQk72rZ9CGzd -cqig2za/BS3zawZWgbv2vB2elNsIllYLdir45jxBOxx2yvJvEuu4glz78y4oJTCTAYAbMlle -5gVdPkVcGyvvVS9tinnSaiIzuvWrYHKWll1uYPm2Q1CDs06P5I7bUGAXpgQLUh/XQguy/0sX -GWqW3FS5JzP+XgcR/7UASvwBdHylubKbeqEpB7G1s+m+8C67qOrc7EQv3Jmy1YDOkhEyNig1 -rmjplLuir3tC1X+D7dHpn7NJe7nMwFx2b2MpMkLA9jPPAGPp/ekcu5sxCe+E0J/4UF++K+CR -XIxgtzU2UJfp8p9x+ygbx5qHinR0tVRdIzv3ZnGsXrfxnWfSOaB582cU3VRN9INzHHax8ETa -QVDnGO5uQa+FiQI8BBgBCAAmAhsMFiEErpbtlp5HmwCE8+F/6I0zNPpfagoFAmEAELYFCQyc -mN4ACgkQ6I0zNPpfagoqAQ/+MnDjBx8JWMd/XjeFoYKx/Oo0ntkInV+ME61JTBls4PdVk+TB -8PWZdPQHw9SnTvRmykFeznXIRzuxkowjrZYXdPXBxY2b1WyD5V3Ati1TM9vqpaR4osyPs2xy -I4dzDssh9YvUsIRL99O04/65lGiYeBNuACq+yK/7nD/ErzBkDYJHhMCdadbVWUACxvVIDvro -yQeVLKMsHqMCd8BTGD7VDs79NXskPnN77pAFnkzS4Z2b8SNzrlgTc5pUiuZHIXPIpEYmsYzh -ucTU6uI3dN1PbSFHK5tG2pHb4ZrPxY3L20Dgc2Tfu5/SDApZzwvvKTqjdO891MEJ++H+ssOz -i4O1UeWKs9owWttan9+PI47ozBSKOTxmMqLSQ0f56Np9FJsV0ilGxRKfjhzJ4KniOMUBA7mP -+m+TmXfVtthJred4sHlJMTJNpt+sCcT6wLMmyc3keIEAu33gsJj3LTpkEA2q+V+ZiP6Q8HRB -402ITklABSArrPSE/fQU9L8hZ5qmy0Z96z0iyILgVMLuRCCfQOMWhwl8yQWIIaf1yPI07xur -epy6lH7HmxjjOR7eo0DaSxQGQpThAtFGwkWkFh8yki8j3E42kkrxvEyyYZDXn2YcI3bpqhJx -PtwCMZUJ3kc/skOrs6bOI19iBNaEoNX5Dllm7UHjOgWNDQkcCuOCxucKano= -=arte ------END PGP PUBLIC KEY BLOCK------ -``` +## Polygon security contact details +security@polygon.technology diff --git a/build/ci.go b/build/ci.go index c3dccfc588..afff1b7328 100644 --- a/build/ci.go +++ b/build/ci.go @@ -24,19 +24,18 @@ Usage: go run build/ci.go Available commands are: - install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables - test [ -coverage ] [ packages... ] -- runs the tests - lint -- runs certain pre-selected linters - archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts - importkeys -- imports signing keys from env - debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package - nsis -- creates a Windows NSIS installer - aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive - xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework - purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore + install [ -arch architecture ] [ -cc compiler ] [ packages... ] -- builds packages and executables + test [ -coverage ] [ packages... ] -- runs the tests + lint -- runs certain pre-selected linters + archive [ -arch architecture ] [ -type zip|tar ] [ -signer key-envvar ] [ -signify key-envvar ] [ -upload dest ] -- archives build artifacts + importkeys -- imports signing keys from env + debsrc [ -signer key-id ] [ -upload dest ] -- creates a debian source package + nsis -- creates a Windows NSIS installer + aar [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an Android archive + xcode [ -local ] [ -sign key-id ] [-deploy repo] [ -upload dest ] -- creates an iOS XCode framework + purge [ -store blobstore ] [ -days threshold ] -- purges old archives from the blobstore For all commands, -n prevents execution of external programs (dry run mode). - */ package main @@ -59,6 +58,7 @@ import ( "time" "github.com/cespare/cp" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/signify" "github.com/ethereum/go-ethereum/internal/build" "github.com/ethereum/go-ethereum/params" @@ -674,21 +674,27 @@ func doDebianSource(cmdline []string) { meta := newDebMetadata(distro, goboot, *signer, env, now, pkg.Name, pkg.Version, pkg.Executables) pkgdir := stageDebianSource(*workdir, meta) + canonicalPath, err := common.VerifyPath(pkgdir) + if err != nil { + fmt.Println("path not verified: " + err.Error()) + return + } + // Add Go source code - if err := build.ExtractArchive(gobundle, pkgdir); err != nil { + if err := build.ExtractArchive(gobundle, canonicalPath); err != nil { log.Fatalf("Failed to extract Go sources: %v", err) } - if err := os.Rename(filepath.Join(pkgdir, "go"), filepath.Join(pkgdir, ".go")); err != nil { + if err := os.Rename(filepath.Join(canonicalPath, "go"), filepath.Join(canonicalPath, ".go")); err != nil { log.Fatalf("Failed to rename Go source folder: %v", err) } // Add all dependency modules in compressed form - os.MkdirAll(filepath.Join(pkgdir, ".mod", "cache"), 0755) - if err := cp.CopyAll(filepath.Join(pkgdir, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { + os.MkdirAll(filepath.Join(canonicalPath, ".mod", "cache"), 0755) + if err := cp.CopyAll(filepath.Join(canonicalPath, ".mod", "cache", "download"), filepath.Join(*workdir, "modgopath", "pkg", "mod", "cache", "download")); err != nil { log.Fatalf("Failed to copy Go module dependencies: %v", err) } // Run the packaging and upload to the PPA debuild := exec.Command("debuild", "-S", "-sa", "-us", "-uc", "-d", "-Zxz", "-nc") - debuild.Dir = pkgdir + debuild.Dir = canonicalPath build.MustRun(debuild) var ( diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 9a251f7884..67710eaeb4 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -162,9 +162,16 @@ func main() { } } // Load up the account key and decrypt its password - blob, err := ioutil.ReadFile(*accPassFlag) + + canonicalPath, err := common.VerifyPath(*accPassFlag) if err != nil { - log.Crit("Failed to read account password contents", "file", *accPassFlag, "err", err) + fmt.Println("path not verified: " + err.Error()) + return + } + + blob, err := ioutil.ReadFile(canonicalPath) + if err != nil { + log.Crit("Failed to read account password contents", "file", canonicalPath, "err", err) } pass := strings.TrimSuffix(string(blob), "\n") diff --git a/common/path.go b/common/path.go index 69820cfe5d..46239d17f7 100644 --- a/common/path.go +++ b/common/path.go @@ -47,3 +47,32 @@ func AbsolutePath(datadir string, filename string) string { } return filepath.Join(datadir, filename) } + +// VerifyPath sanitizes the path to avoid Path Traversal vulnerability +func VerifyPath(path string) (string, error) { + c := filepath.Clean(path) + + r, err := filepath.EvalSymlinks(c) + if err != nil { + return c, fmt.Errorf("unsafe or invalid path specified: %s", path) + } else { + return r, nil + } +} + +// VerifyCrasher sanitizes the path to avoid Path Traversal vulnerability and reads the file from that path, returning its content +func VerifyCrasher(crasher string) []byte { + canonicalPath, err := VerifyPath(crasher) + if err != nil { + fmt.Println("path not verified: " + err.Error()) + return nil + } + + data, err := os.ReadFile(canonicalPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", canonicalPath, err) + os.Exit(1) + } + + return data +} diff --git a/go.mod b/go.mod index 36595ca307..f770311c31 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 - golang.org/x/text v0.3.7 + golang.org/x/text v0.3.8 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.12 gonum.org/v1/gonum v0.11.0 @@ -141,3 +141,5 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 diff --git a/go.sum b/go.sum index 96fa9d3f04..4403b347d2 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/JekaMas/go-grpc-net-conn v0.0.0-20220708155319-6aff21f2d13d h1:RO27lgfZF8s9lZ3pWyzc0gCE0RZC+6/PXbRjAa0CNp8= github.com/JekaMas/go-grpc-net-conn v0.0.0-20220708155319-6aff21f2d13d/go.mod h1:romz7UPgSYhfJkKOalzEEyV6sWtt/eAEm0nX2aOrod0= -github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= -github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.22.0+incompatible h1:z4yfnGrZ7netVz+0EDJ0Wi+5VZCSYp4Z0m2dk6cEM60= @@ -667,8 +667,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/metrics/metrics.go b/metrics/metrics.go index 1d0133e850..e54bb3e0d2 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -6,11 +6,14 @@ package metrics import ( + "fmt" "os" "runtime" "strings" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/BurntSushi/toml" ) @@ -71,7 +74,13 @@ func init() { func updateMetricsFromConfig(path string) { // Don't act upon any errors here. They're already taken into // consideration when the toml config file will be parsed in the cli. - data, err := os.ReadFile(path) + canonicalPath, err := common.VerifyPath(path) + if err != nil { + fmt.Println("path not verified: " + err.Error()) + return + } + + data, err := os.ReadFile(canonicalPath) tomlData := string(data) if err != nil { diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 5b240bfd85..cfee358c9d 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -26,6 +26,8 @@ import ( "os" "golang.org/x/tools/go/packages" + + "github.com/ethereum/go-ethereum/common" ) const pathOfPackageRLP = "github.com/ethereum/go-ethereum/rlp" @@ -52,8 +54,15 @@ func main() { } if *output == "-" { os.Stdout.Write(code) - } else if err := ioutil.WriteFile(*output, code, 0644); err != nil { - fatal(err) + } else { + canonicalPath, err := common.VerifyPath(*output) + if err != nil { + fmt.Println("path not verified: " + err.Error()) + fatal(err) + } + if err := ioutil.WriteFile(canonicalPath, code, 0600); err != nil { + fatal(err) + } } } diff --git a/scripts/getconfig.go b/scripts/getconfig.go index caf3f45a8e..665bd0d2a3 100644 --- a/scripts/getconfig.go +++ b/scripts/getconfig.go @@ -11,6 +11,7 @@ import ( "github.com/pelletier/go-toml" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/internal/cli/server" ) @@ -514,7 +515,13 @@ func commentFlags(path string, updatedArgs []string) { ignoreLineFlag := false - input, err := os.ReadFile(path) + canonicalPath, err := common.VerifyPath(path) + if err != nil { + fmt.Println("path not verified: " + err.Error()) + return + } + + input, err := os.ReadFile(canonicalPath) if err != nil { log.Fatalln(err) } @@ -594,7 +601,7 @@ func commentFlags(path string, updatedArgs []string) { output := strings.Join(newLines, "\n") - err = os.WriteFile(path, []byte(output), 0600) + err = os.WriteFile(canonicalPath, []byte(output), 0600) if err != nil { log.Fatalln(err) } diff --git a/tests/fuzzers/difficulty/debug/main.go b/tests/fuzzers/difficulty/debug/main.go index 23516b3a0d..0bd4478949 100644 --- a/tests/fuzzers/difficulty/debug/main.go +++ b/tests/fuzzers/difficulty/debug/main.go @@ -2,9 +2,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/tests/fuzzers/difficulty" ) @@ -14,10 +14,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + difficulty.Fuzz(data) } diff --git a/tests/fuzzers/les/debug/main.go b/tests/fuzzers/les/debug/main.go index 09e087d4c8..c4b8803954 100644 --- a/tests/fuzzers/les/debug/main.go +++ b/tests/fuzzers/les/debug/main.go @@ -18,9 +18,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/tests/fuzzers/les" ) @@ -32,10 +32,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + les.Fuzz(data) } diff --git a/tests/fuzzers/rangeproof/debug/main.go b/tests/fuzzers/rangeproof/debug/main.go index a81c69fea5..9e782c6dda 100644 --- a/tests/fuzzers/rangeproof/debug/main.go +++ b/tests/fuzzers/rangeproof/debug/main.go @@ -18,9 +18,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/tests/fuzzers/rangeproof" ) @@ -32,10 +32,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + rangeproof.Fuzz(data) } diff --git a/tests/fuzzers/snap/debug/main.go b/tests/fuzzers/snap/debug/main.go index d0d1b49307..d7f8a4a9f2 100644 --- a/tests/fuzzers/snap/debug/main.go +++ b/tests/fuzzers/snap/debug/main.go @@ -18,9 +18,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/tests/fuzzers/snap" ) @@ -30,10 +30,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + snap.FuzzTrieNodes(data) } diff --git a/tests/fuzzers/stacktrie/debug/main.go b/tests/fuzzers/stacktrie/debug/main.go index 1ec28a8ef1..b7dbafbcc5 100644 --- a/tests/fuzzers/stacktrie/debug/main.go +++ b/tests/fuzzers/stacktrie/debug/main.go @@ -2,9 +2,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/tests/fuzzers/stacktrie" ) @@ -14,10 +14,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + stacktrie.Debug(data) } diff --git a/tests/fuzzers/vflux/debug/main.go b/tests/fuzzers/vflux/debug/main.go index 1d4a5ff19c..ed992752a3 100644 --- a/tests/fuzzers/vflux/debug/main.go +++ b/tests/fuzzers/vflux/debug/main.go @@ -18,9 +18,9 @@ package main import ( "fmt" - "io/ioutil" "os" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/tests/fuzzers/vflux" ) @@ -35,10 +35,11 @@ func main() { os.Exit(1) } crasher := os.Args[1] - data, err := ioutil.ReadFile(crasher) - if err != nil { - fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) - os.Exit(1) + + data := common.VerifyCrasher(crasher) + if data == nil { + return } + vflux.FuzzClientPool(data) } From a75eb71e4823430cc693f9d9385fc74b23988356 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 12 Dec 2022 02:52:50 +0530 Subject: [PATCH 61/70] fix linters --- internal/cli/server/config_test.go | 6 ++++++ miner/worker.go | 3 +++ 2 files changed, 9 insertions(+) diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index 5f3118996b..c444ee7b98 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -102,6 +102,8 @@ func TestDefaultDatatypeOverride(t *testing.T) { } func TestConfigLoadFile(t *testing.T) { + t.Parallel() + readFile := func(path string) { config, err := readConfigFile(path) assert.NoError(t, err) @@ -124,11 +126,15 @@ func TestConfigLoadFile(t *testing.T) { // read file in hcl format t.Run("hcl", func(t *testing.T) { + t.Parallel() + readFile("./testdata/test.hcl") }) // read file in json format t.Run("json", func(t *testing.T) { + t.Parallel() + readFile("./testdata/test.json") }) } diff --git a/miner/worker.go b/miner/worker.go index 2c8cb3f790..7876c18c8a 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -920,6 +920,7 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP initialGasLimit := env.gasPool.Gas() initialTxs := txs.GetTxs() + var breakCause string defer func() { @@ -950,12 +951,14 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP inc: true, } } + breakCause = "interrupt" return atomic.LoadInt32(interrupt) == commitInterruptNewHead } // If we don't have enough gas for any further transactions then we're done if env.gasPool.Gas() < params.TxGas { log.Trace("Not enough gas for further transactions", "have", env.gasPool, "want", params.TxGas) + breakCause = "Not enough gas for further transactions" break } From f261c933d5805f31f485738cdafea2fdcd6d3971 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 12 Dec 2022 12:07:27 +0530 Subject: [PATCH 62/70] upgrade grpc version --- go.mod | 4 ++-- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f770311c31..b5a84d0bd1 100644 --- a/go.mod +++ b/go.mod @@ -71,11 +71,11 @@ require ( golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 - golang.org/x/text v0.3.8 + golang.org/x/text v0.4.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.1.12 gonum.org/v1/gonum v0.11.0 - google.golang.org/grpc v1.48.0 + google.golang.org/grpc v1.51.0 google.golang.org/protobuf v1.28.0 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 diff --git a/go.sum b/go.sum index 4403b347d2..a37a00a6dd 100644 --- a/go.sum +++ b/go.sum @@ -105,7 +105,6 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= @@ -147,7 +146,6 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -668,8 +666,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -765,8 +763,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 469bd2918c08b395c2eed8109f7eb6176c93c9fb Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Mon, 12 Dec 2022 12:26:12 +0530 Subject: [PATCH 63/70] add ignore rule for net/http2 --- .snyk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.snyk b/.snyk index 2fa83cf27c..285024f5bb 100644 --- a/.snyk +++ b/.snyk @@ -34,4 +34,8 @@ ignore: - '*': reason: 'As open source org, we have no issues with licenses' created: 2022-11-11T08:09:41.635Z + 'SNYK-GOLANG-GOLANGORGXNETHTTP2-3160322': + - '*': + reason: 'grpc working on a release to fix the issue' + created: 2022-12-12T06:50:00.000Z patch: {} From b1d86bd6ea8bee640291e8d7ee7e782027bb38f9 Mon Sep 17 00:00:00 2001 From: SHIVAM SHARMA Date: Mon, 12 Dec 2022 12:34:02 +0530 Subject: [PATCH 64/70] Shivam/txpool tracing (#604) * lock, unlock to rlock, runlock * add : tracing Pending() and Locals() * Log time spent in committing a tx during mining * Remove data from logging * Move log into case where a tx completes without error * profile fillTransactions * fix conflict * bug fixes * add logs * txpool: add tracing in Pending() * rearrange tracing * add attributes * fix * fix * log error in profiling * update file mode and file path for profiling * full profiling * fix log * fix log * less wait * fix * fix * logs * worker: use block number for prof files * initial * txList add * fix gas calculation * fix * green tests * linters * prettify * allocate less * no locks between pending and reorg * no locks * no locks on locals * more tests * linters * less allocs * comment * optimize errors * linters * fix * fix * Linters * linters * linters * simplify errors * atomics for transactions * fix * optimize workers * fix copy * linters * txpool tracing * linters * fix tracing * duration in mcs * locks * metrics * fix * cache hit/miss * less locks on evict * remove once * remove once * switch off pprof * fix data race * fix data race * add : sealed total/empty blocks metric gauge * add : RPC debug_getTraceStack * fix : RPC debug_getTraceStack * fix : RPC debug_getTraceStack for all go-routines * linters * add data race test on txpool * fix concurrency * noleak * increase batch size * prettify * tests * baseFee mutex * panic fix * linters * fix gas fee data race * linters * more transactions * debug * debug * fix ticker * fix test * add cacheMu * more tests * fix test panic * linters * add statistics * add statistics * txitems data race * fix tx list Has * fix : lint Co-authored-by: Arpit Temani Co-authored-by: Jerry Co-authored-by: Manav Darji Co-authored-by: Evgeny Danienko <6655321@bk.ru> --- Makefile | 7 +- cmd/evm/internal/t8ntool/transaction.go | 3 +- common/debug/debug.go | 24 + common/math/big.go | 29 +- common/math/uint.go | 23 + common/time.go | 9 + common/tracing/context.go | 10 +- consensus/bor/bor.go | 8 +- consensus/misc/eip1559.go | 53 + core/tx_journal.go | 17 +- core/tx_list.go | 236 +++- core/tx_list_test.go | 9 +- core/tx_pool.go | 1068 +++++++++++---- core/tx_pool_test.go | 1599 ++++++++++++++++++++++- core/types/access_list_tx.go | 65 +- core/types/dynamic_fee_tx.go | 71 +- core/types/legacy_tx.go | 60 +- core/types/transaction.go | 180 ++- core/types/transaction_signing.go | 13 +- core/types/transaction_test.go | 32 +- eth/api_backend.go | 11 +- eth/bor_checkpoint_verifier.go | 1 + eth/handler.go | 3 +- eth/handler_test.go | 3 +- eth/sync.go | 7 +- internal/cli/server/pprof/pprof.go | 22 + internal/ethapi/api.go | 16 + internal/web3ext/web3ext.go | 5 + les/handler_test.go | 2 +- les/server_requests.go | 22 +- miner/worker.go | 192 ++- tests/init_test.go | 3 - 32 files changed, 3352 insertions(+), 451 deletions(-) create mode 100644 common/math/uint.go create mode 100644 common/time.go diff --git a/Makefile b/Makefile index e51d2d99eb..f0f9385e7b 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,10 @@ ios: @echo "Import \"$(GOBIN)/Geth.framework\" to use the library." test: - $(GOTEST) --timeout 5m -shuffle=on -cover -coverprofile=cover.out $(TESTALL) + $(GOTEST) --timeout 5m -shuffle=on -cover -short -coverprofile=cover.out -covermode=atomic $(TESTALL) + +test-txpool-race: + $(GOTEST) -run=TestPoolMiningDataRaces --timeout 600m -race -v ./core/ test-race: $(GOTEST) --timeout 15m -race -shuffle=on $(TESTALL) @@ -75,7 +78,7 @@ lint: lintci-deps: rm -f ./build/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.48.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.50.1 goimports: goimports -local "$(PACKAGE)" -w . diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 6f1c964ada..cf2039b66c 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -24,6 +24,8 @@ import ( "os" "strings" + "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" @@ -32,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - "gopkg.in/urfave/cli.v1" ) type result struct { diff --git a/common/debug/debug.go b/common/debug/debug.go index 6a677e495d..056ebe2fa7 100644 --- a/common/debug/debug.go +++ b/common/debug/debug.go @@ -1,6 +1,7 @@ package debug import ( + "fmt" "runtime" ) @@ -26,3 +27,26 @@ func Callers(show int) []string { return callers } + +func CodeLine() (string, string, int) { + pc, filename, line, _ := runtime.Caller(1) + return runtime.FuncForPC(pc).Name(), filename, line +} + +func CodeLineStr() string { + pc, filename, line, _ := runtime.Caller(1) + return fmt.Sprintf("%s:%d - %s", filename, line, runtime.FuncForPC(pc).Name()) +} + +func Stack(all bool) []byte { + buf := make([]byte, 4096) + + for { + n := runtime.Stack(buf, all) + if n < len(buf) { + return buf[:n] + } + + buf = make([]byte, 2*len(buf)) + } +} diff --git a/common/math/big.go b/common/math/big.go index 1af5b4d879..4ccf89e38c 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -20,6 +20,8 @@ package math import ( "fmt" "math/big" + + "github.com/holiman/uint256" ) // Various big integer limit values. @@ -132,6 +134,7 @@ func MustParseBig256(s string) *big.Int { // BigPow returns a ** b as a big integer. func BigPow(a, b int64) *big.Int { r := big.NewInt(a) + return r.Exp(r, big.NewInt(b), nil) } @@ -140,6 +143,15 @@ func BigMax(x, y *big.Int) *big.Int { if x.Cmp(y) < 0 { return y } + + return x +} + +func BigMaxUint(x, y *uint256.Int) *uint256.Int { + if x.Lt(y) { + return y + } + return x } @@ -148,6 +160,15 @@ func BigMin(x, y *big.Int) *big.Int { if x.Cmp(y) > 0 { return y } + + return x +} + +func BigMinUint256(x, y *uint256.Int) *uint256.Int { + if x.Gt(y) { + return y + } + return x } @@ -227,10 +248,10 @@ func U256Bytes(n *big.Int) []byte { // S256 interprets x as a two's complement number. // x must not exceed 256 bits (the result is undefined if it does) and is not modified. // -// S256(0) = 0 -// S256(1) = 1 -// S256(2**255) = -2**255 -// S256(2**256-1) = -1 +// S256(0) = 0 +// S256(1) = 1 +// S256(2**255) = -2**255 +// S256(2**256-1) = -1 func S256(x *big.Int) *big.Int { if x.Cmp(tt255) < 0 { return x diff --git a/common/math/uint.go b/common/math/uint.go new file mode 100644 index 0000000000..96b8261884 --- /dev/null +++ b/common/math/uint.go @@ -0,0 +1,23 @@ +package math + +import ( + "math/big" + + "github.com/holiman/uint256" +) + +var ( + U0 = uint256.NewInt(0) + U1 = uint256.NewInt(1) + U100 = uint256.NewInt(100) +) + +func U256LTE(a, b *uint256.Int) bool { + return a.Lt(b) || a.Eq(b) +} + +func FromBig(v *big.Int) *uint256.Int { + u, _ := uint256.FromBig(v) + + return u +} diff --git a/common/time.go b/common/time.go new file mode 100644 index 0000000000..6c7662e04c --- /dev/null +++ b/common/time.go @@ -0,0 +1,9 @@ +package common + +import "time" + +const TimeMilliseconds = "15:04:05.000" + +func NowMilliseconds() string { + return time.Now().Format(TimeMilliseconds) +} diff --git a/common/tracing/context.go b/common/tracing/context.go index 510e45d775..c3c6342502 100644 --- a/common/tracing/context.go +++ b/common/tracing/context.go @@ -4,6 +4,7 @@ import ( "context" "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) @@ -51,11 +52,16 @@ func Trace(ctx context.Context, spanName string) (context.Context, trace.Span) { return tr.Start(ctx, spanName) } -func Exec(ctx context.Context, spanName string, opts ...Option) { +func Exec(ctx context.Context, instrumentationName, spanName string, opts ...Option) { var span trace.Span tr := FromContext(ctx) + if tr == nil && len(instrumentationName) != 0 { + tr = otel.GetTracerProvider().Tracer(instrumentationName) + ctx = WithTracer(ctx, tr) + } + if tr != nil { ctx, span = tr.Start(ctx, spanName) } @@ -85,7 +91,7 @@ func ElapsedTime(ctx context.Context, span trace.Span, msg string, fn func(conte fn(ctx, span) if span != nil { - span.SetAttributes(attribute.Int(msg, int(time.Since(now).Milliseconds()))) + span.SetAttributes(attribute.Int(msg, int(time.Since(now).Microseconds()))) } } diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 1b4ddec45d..b6d643eeba 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -821,7 +821,7 @@ func (c *Bor) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHead if IsSprintStart(headerNumber, c.config.CalculateSprint(headerNumber)) { cx := statefull.ChainContext{Chain: chain, Bor: c} - tracing.Exec(finalizeCtx, "bor.checkAndCommitSpan", func(ctx context.Context, span trace.Span) { + tracing.Exec(finalizeCtx, "", "bor.checkAndCommitSpan", func(ctx context.Context, span trace.Span) { // check and commit span err = c.checkAndCommitSpan(finalizeCtx, state, header, cx) }) @@ -832,7 +832,7 @@ func (c *Bor) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHead } if c.HeimdallClient != nil { - tracing.Exec(finalizeCtx, "bor.checkAndCommitSpan", func(ctx context.Context, span trace.Span) { + tracing.Exec(finalizeCtx, "", "bor.checkAndCommitSpan", func(ctx context.Context, span trace.Span) { // commit states stateSyncData, err = c.CommitStates(finalizeCtx, state, header, cx) }) @@ -844,7 +844,7 @@ func (c *Bor) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHead } } - tracing.Exec(finalizeCtx, "bor.changeContractCodeIfNeeded", func(ctx context.Context, span trace.Span) { + tracing.Exec(finalizeCtx, "", "bor.changeContractCodeIfNeeded", func(ctx context.Context, span trace.Span) { err = c.changeContractCodeIfNeeded(headerNumber, state) }) @@ -854,7 +854,7 @@ func (c *Bor) FinalizeAndAssemble(ctx context.Context, chain consensus.ChainHead } // No block rewards in PoA, so the state remains as it is - tracing.Exec(finalizeCtx, "bor.IntermediateRoot", func(ctx context.Context, span trace.Span) { + tracing.Exec(finalizeCtx, "", "bor.IntermediateRoot", func(ctx context.Context, span trace.Span) { header.Root = state.IntermediateRoot(chain.Config().IsEIP158(header.Number)) }) diff --git a/consensus/misc/eip1559.go b/consensus/misc/eip1559.go index 193a5b84e2..00a8ab5b58 100644 --- a/consensus/misc/eip1559.go +++ b/consensus/misc/eip1559.go @@ -20,6 +20,8 @@ import ( "fmt" "math/big" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" @@ -92,3 +94,54 @@ func CalcBaseFee(config *params.ChainConfig, parent *types.Header) *big.Int { ) } } + +// CalcBaseFee calculates the basefee of the header. +func CalcBaseFeeUint(config *params.ChainConfig, parent *types.Header) *uint256.Int { + var ( + initialBaseFeeUint = uint256.NewInt(params.InitialBaseFee) + baseFeeChangeDenominatorUint64 = params.BaseFeeChangeDenominator(config.Bor, parent.Number) + baseFeeChangeDenominatorUint = uint256.NewInt(baseFeeChangeDenominatorUint64) + ) + + // If the current block is the first EIP-1559 block, return the InitialBaseFee. + if !config.IsLondon(parent.Number) { + return initialBaseFeeUint.Clone() + } + + var ( + parentGasTarget = parent.GasLimit / params.ElasticityMultiplier + parentGasTargetBig = uint256.NewInt(parentGasTarget) + ) + + // If the parent gasUsed is the same as the target, the baseFee remains unchanged. + if parent.GasUsed == parentGasTarget { + return math.FromBig(parent.BaseFee) + } + + if parent.GasUsed > parentGasTarget { + // If the parent block used more gas than its target, the baseFee should increase. + gasUsedDelta := uint256.NewInt(parent.GasUsed - parentGasTarget) + + parentBaseFee := math.FromBig(parent.BaseFee) + x := gasUsedDelta.Mul(parentBaseFee, gasUsedDelta) + y := x.Div(x, parentGasTargetBig) + baseFeeDelta := math.BigMaxUint( + x.Div(y, baseFeeChangeDenominatorUint), + math.U1, + ) + + return x.Add(parentBaseFee, baseFeeDelta) + } + + // Otherwise if the parent block used less gas than its target, the baseFee should decrease. + gasUsedDelta := uint256.NewInt(parentGasTarget - parent.GasUsed) + parentBaseFee := math.FromBig(parent.BaseFee) + x := gasUsedDelta.Mul(parentBaseFee, gasUsedDelta) + y := x.Div(x, parentGasTargetBig) + baseFeeDelta := x.Div(y, baseFeeChangeDenominatorUint) + + return math.BigMaxUint( + x.Sub(parentBaseFee, baseFeeDelta), + math.U0.Clone(), + ) +} diff --git a/core/tx_journal.go b/core/tx_journal.go index d282126a08..980bdb9864 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -61,11 +61,13 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { if _, err := os.Stat(journal.path); os.IsNotExist(err) { return nil } + // Open the journal for loading any past transactions input, err := os.Open(journal.path) if err != nil { return err } + defer input.Close() // Temporarily discard any journal additions (don't double add on load) @@ -80,29 +82,35 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { // appropriate progress counters. Then use this method to load all the // journaled transactions in small-ish batches. loadBatch := func(txs types.Transactions) { + errs := add(txs) + + dropped = len(errs) + for _, err := range add(txs) { - if err != nil { - log.Debug("Failed to add journaled transaction", "err", err) - dropped++ - } + log.Debug("Failed to add journaled transaction", "err", err) } } var ( failure error batch types.Transactions ) + for { // Parse the next transaction and terminate on error tx := new(types.Transaction) + if err = stream.Decode(tx); err != nil { if err != io.EOF { failure = err } + if batch.Len() > 0 { loadBatch(batch) } + break } + // New transaction parsed, queue up for later, import if threshold is reached total++ @@ -111,6 +119,7 @@ func (journal *txJournal) load(add func([]*types.Transaction) []error) error { batch = batch[:0] } } + log.Info("Loaded local transaction journal", "transactions", total, "dropped", dropped) return failure diff --git a/core/tx_list.go b/core/tx_list.go index f141a03bbd..e763777e33 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -19,13 +19,15 @@ package core import ( "container/heap" "math" - "math/big" "sort" "sync" "sync/atomic" "time" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" + cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" ) @@ -54,36 +56,67 @@ func (h *nonceHeap) Pop() interface{} { type txSortedMap struct { items map[uint64]*types.Transaction // Hash map storing the transaction data index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode) - cache types.Transactions // Cache of the transactions already sorted + m sync.RWMutex + + cache types.Transactions // Cache of the transactions already sorted + isEmpty bool + cacheMu sync.RWMutex } // newTxSortedMap creates a new nonce-sorted transaction map. func newTxSortedMap() *txSortedMap { return &txSortedMap{ - items: make(map[uint64]*types.Transaction), - index: new(nonceHeap), + items: make(map[uint64]*types.Transaction), + index: new(nonceHeap), + isEmpty: true, } } // Get retrieves the current transactions associated with the given nonce. func (m *txSortedMap) Get(nonce uint64) *types.Transaction { + m.m.RLock() + defer m.m.RUnlock() + return m.items[nonce] } +func (m *txSortedMap) Has(nonce uint64) bool { + if m == nil { + return false + } + + m.m.RLock() + defer m.m.RUnlock() + + return m.items[nonce] != nil +} + // Put inserts a new transaction into the map, also updating the map's nonce // index. If a transaction already exists with the same nonce, it's overwritten. func (m *txSortedMap) Put(tx *types.Transaction) { + m.m.Lock() + defer m.m.Unlock() + nonce := tx.Nonce() if m.items[nonce] == nil { heap.Push(m.index, nonce) } - m.items[nonce], m.cache = tx, nil + + m.items[nonce] = tx + + m.cacheMu.Lock() + m.isEmpty = true + m.cache = nil + m.cacheMu.Unlock() } // Forward removes all transactions from the map with a nonce lower than the // provided threshold. Every removed transaction is returned for any post-removal // maintenance. func (m *txSortedMap) Forward(threshold uint64) types.Transactions { + m.m.Lock() + defer m.m.Unlock() + var removed types.Transactions // Pop off heap items until the threshold is reached @@ -92,10 +125,15 @@ func (m *txSortedMap) Forward(threshold uint64) types.Transactions { removed = append(removed, m.items[nonce]) delete(m.items, nonce) } + // If we had a cached order, shift the front + m.cacheMu.Lock() if m.cache != nil { + hitCacheCounter.Inc(1) m.cache = m.cache[len(removed):] } + m.cacheMu.Unlock() + return removed } @@ -105,6 +143,9 @@ func (m *txSortedMap) Forward(threshold uint64) types.Transactions { // If you want to do several consecutive filterings, it's therefore better to first // do a .filter(func1) followed by .Filter(func2) or reheap() func (m *txSortedMap) Filter(filter func(*types.Transaction) bool) types.Transactions { + m.m.Lock() + defer m.m.Unlock() + removed := m.filter(filter) // If transactions were removed, the heap and cache are ruined if len(removed) > 0 { @@ -115,11 +156,19 @@ func (m *txSortedMap) Filter(filter func(*types.Transaction) bool) types.Transac func (m *txSortedMap) reheap() { *m.index = make([]uint64, 0, len(m.items)) + for nonce := range m.items { *m.index = append(*m.index, nonce) } + heap.Init(m.index) + + m.cacheMu.Lock() m.cache = nil + m.isEmpty = true + m.cacheMu.Unlock() + + resetCacheGauge.Inc(1) } // filter is identical to Filter, but **does not** regenerate the heap. This method @@ -135,7 +184,12 @@ func (m *txSortedMap) filter(filter func(*types.Transaction) bool) types.Transac } } if len(removed) > 0 { + m.cacheMu.Lock() m.cache = nil + m.isEmpty = true + m.cacheMu.Unlock() + + resetCacheGauge.Inc(1) } return removed } @@ -143,45 +197,66 @@ func (m *txSortedMap) filter(filter func(*types.Transaction) bool) types.Transac // Cap places a hard limit on the number of items, returning all transactions // exceeding that limit. func (m *txSortedMap) Cap(threshold int) types.Transactions { + m.m.Lock() + defer m.m.Unlock() + // Short circuit if the number of items is under the limit if len(m.items) <= threshold { return nil } + // Otherwise gather and drop the highest nonce'd transactions var drops types.Transactions sort.Sort(*m.index) + for size := len(m.items); size > threshold; size-- { drops = append(drops, m.items[(*m.index)[size-1]]) delete(m.items, (*m.index)[size-1]) } + *m.index = (*m.index)[:threshold] heap.Init(m.index) // If we had a cache, shift the back + m.cacheMu.Lock() if m.cache != nil { m.cache = m.cache[:len(m.cache)-len(drops)] } + m.cacheMu.Unlock() + return drops } // Remove deletes a transaction from the maintained map, returning whether the // transaction was found. func (m *txSortedMap) Remove(nonce uint64) bool { + m.m.Lock() + defer m.m.Unlock() + // Short circuit if no transaction is present _, ok := m.items[nonce] if !ok { return false } + // Otherwise delete the transaction and fix the heap index for i := 0; i < m.index.Len(); i++ { if (*m.index)[i] == nonce { heap.Remove(m.index, i) + break } } + delete(m.items, nonce) + + m.cacheMu.Lock() m.cache = nil + m.isEmpty = true + m.cacheMu.Unlock() + + resetCacheGauge.Inc(1) return true } @@ -194,55 +269,125 @@ func (m *txSortedMap) Remove(nonce uint64) bool { // prevent getting into and invalid state. This is not something that should ever // happen but better to be self correcting than failing! func (m *txSortedMap) Ready(start uint64) types.Transactions { + m.m.Lock() + defer m.m.Unlock() + // Short circuit if no transactions are available if m.index.Len() == 0 || (*m.index)[0] > start { return nil } + // Otherwise start accumulating incremental transactions var ready types.Transactions + for next := (*m.index)[0]; m.index.Len() > 0 && (*m.index)[0] == next; next++ { ready = append(ready, m.items[next]) delete(m.items, next) heap.Pop(m.index) } + + m.cacheMu.Lock() m.cache = nil + m.isEmpty = true + m.cacheMu.Unlock() + + resetCacheGauge.Inc(1) return ready } // Len returns the length of the transaction map. func (m *txSortedMap) Len() int { + m.m.RLock() + defer m.m.RUnlock() + return len(m.items) } func (m *txSortedMap) flatten() types.Transactions { // If the sorting was not cached yet, create and cache it - if m.cache == nil { - m.cache = make(types.Transactions, 0, len(m.items)) + m.cacheMu.Lock() + defer m.cacheMu.Unlock() + + if m.isEmpty { + m.isEmpty = false // to simulate sync.Once + + m.cacheMu.Unlock() + + m.m.RLock() + + cache := make(types.Transactions, 0, len(m.items)) + for _, tx := range m.items { - m.cache = append(m.cache, tx) + cache = append(cache, tx) } - sort.Sort(types.TxByNonce(m.cache)) + + m.m.RUnlock() + + // exclude sorting from locks + sort.Sort(types.TxByNonce(cache)) + + m.cacheMu.Lock() + m.cache = cache + + reinitCacheGauge.Inc(1) + missCacheCounter.Inc(1) + } else { + hitCacheCounter.Inc(1) } + return m.cache } +func (m *txSortedMap) lastElement() *types.Transaction { + // If the sorting was not cached yet, create and cache it + m.cacheMu.Lock() + defer m.cacheMu.Unlock() + + cache := m.cache + + if m.isEmpty { + m.isEmpty = false // to simulate sync.Once + + m.cacheMu.Unlock() + + cache = make(types.Transactions, 0, len(m.items)) + + m.m.RLock() + + for _, tx := range m.items { + cache = append(cache, tx) + } + + m.m.RUnlock() + + // exclude sorting from locks + sort.Sort(types.TxByNonce(cache)) + + m.cacheMu.Lock() + m.cache = cache + + reinitCacheGauge.Inc(1) + missCacheCounter.Inc(1) + } else { + hitCacheCounter.Inc(1) + } + + return cache[len(cache)-1] +} + // Flatten creates a nonce-sorted slice of transactions based on the loosely // sorted internal representation. The result of the sorting is cached in case // it's requested again before any modifications are made to the contents. func (m *txSortedMap) Flatten() types.Transactions { // Copy the cache to prevent accidental modifications - cache := m.flatten() - txs := make(types.Transactions, len(cache)) - copy(txs, cache) - return txs + return m.flatten() } // LastElement returns the last element of a flattened list, thus, the // transaction with the highest nonce func (m *txSortedMap) LastElement() *types.Transaction { - cache := m.flatten() - return cache[len(cache)-1] + return m.lastElement() } // txList is a "list" of transactions belonging to an account, sorted by account @@ -253,17 +398,16 @@ type txList struct { strict bool // Whether nonces are strictly continuous or not txs *txSortedMap // Heap indexed sorted hash map of the transactions - costcap *big.Int // Price of the highest costing transaction (reset only if exceeds balance) - gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) + costcap *uint256.Int // Price of the highest costing transaction (reset only if exceeds balance) + gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) } // newTxList create a new transaction list for maintaining nonce-indexable fast, // gapped, sortable transaction lists. func newTxList(strict bool) *txList { return &txList{ - strict: strict, - txs: newTxSortedMap(), - costcap: new(big.Int), + strict: strict, + txs: newTxSortedMap(), } } @@ -285,31 +429,36 @@ func (l *txList) Add(tx *types.Transaction, priceBump uint64) (bool, *types.Tran if old.GasFeeCapCmp(tx) >= 0 || old.GasTipCapCmp(tx) >= 0 { return false, nil } + // thresholdFeeCap = oldFC * (100 + priceBump) / 100 - a := big.NewInt(100 + int64(priceBump)) - aFeeCap := new(big.Int).Mul(a, old.GasFeeCap()) - aTip := a.Mul(a, old.GasTipCap()) + a := uint256.NewInt(100 + priceBump) + aFeeCap := uint256.NewInt(0).Mul(a, old.GasFeeCapUint()) + aTip := a.Mul(a, old.GasTipCapUint()) // thresholdTip = oldTip * (100 + priceBump) / 100 - b := big.NewInt(100) + b := cmath.U100 thresholdFeeCap := aFeeCap.Div(aFeeCap, b) thresholdTip := aTip.Div(aTip, b) // We have to ensure that both the new fee cap and tip are higher than the // old ones as well as checking the percentage threshold to ensure that // this is accurate for low (Wei-level) gas price replacements. - if tx.GasFeeCapIntCmp(thresholdFeeCap) < 0 || tx.GasTipCapIntCmp(thresholdTip) < 0 { + if tx.GasFeeCapUIntLt(thresholdFeeCap) || tx.GasTipCapUIntLt(thresholdTip) { return false, nil } } + // Otherwise overwrite the old transaction with the current one l.txs.Put(tx) - if cost := tx.Cost(); l.costcap.Cmp(cost) < 0 { + + if cost := tx.CostUint(); l.costcap == nil || l.costcap.Lt(cost) { l.costcap = cost } + if gas := tx.Gas(); l.gascap < gas { l.gascap = gas } + return true, old } @@ -329,17 +478,20 @@ func (l *txList) Forward(threshold uint64) types.Transactions { // a point in calculating all the costs or if the balance covers all. If the threshold // is lower than the costgas cap, the caps will be reset to a new high after removing // the newly invalidated transactions. -func (l *txList) Filter(costLimit *big.Int, gasLimit uint64) (types.Transactions, types.Transactions) { +func (l *txList) Filter(costLimit *uint256.Int, gasLimit uint64) (types.Transactions, types.Transactions) { // If all transactions are below the threshold, short circuit - if l.costcap.Cmp(costLimit) <= 0 && l.gascap <= gasLimit { + if cmath.U256LTE(l.costcap, costLimit) && l.gascap <= gasLimit { return nil, nil } - l.costcap = new(big.Int).Set(costLimit) // Lower the caps to the thresholds + + l.costcap = costLimit.Clone() // Lower the caps to the thresholds l.gascap = gasLimit // Filter out all the transactions above the account's funds + cost := uint256.NewInt(0) removed := l.txs.Filter(func(tx *types.Transaction) bool { - return tx.Gas() > gasLimit || tx.Cost().Cmp(costLimit) > 0 + cost.SetFromBig(tx.Cost()) + return tx.Gas() > gasLimit || cost.Gt(costLimit) }) if len(removed) == 0 { @@ -416,13 +568,18 @@ func (l *txList) LastElement() *types.Transaction { return l.txs.LastElement() } +func (l *txList) Has(nonce uint64) bool { + return l != nil && l.txs.items[nonce] != nil +} + // priceHeap is a heap.Interface implementation over transactions for retrieving // price-sorted transactions to discard when the pool fills up. If baseFee is set // then the heap is sorted based on the effective tip based on the given base fee. // If baseFee is nil then the sorting is based on gasFeeCap. type priceHeap struct { - baseFee *big.Int // heap should always be re-sorted after baseFee is changed - list []*types.Transaction + baseFee *uint256.Int // heap should always be re-sorted after baseFee is changed + list []*types.Transaction + baseFeeMu sync.RWMutex } func (h *priceHeap) Len() int { return len(h.list) } @@ -440,16 +597,24 @@ func (h *priceHeap) Less(i, j int) bool { } func (h *priceHeap) cmp(a, b *types.Transaction) int { + h.baseFeeMu.RLock() + if h.baseFee != nil { // Compare effective tips if baseFee is specified - if c := a.EffectiveGasTipCmp(b, h.baseFee); c != 0 { + if c := a.EffectiveGasTipTxUintCmp(b, h.baseFee); c != 0 { + h.baseFeeMu.RUnlock() + return c } } + + h.baseFeeMu.RUnlock() + // Compare fee caps if baseFee is not specified or effective tips are equal if c := a.GasFeeCapCmp(b); c != 0 { return c } + // Compare tips if effective tips and fee caps are equal return a.GasTipCapCmp(b) } @@ -629,7 +794,10 @@ func (l *txPricedList) Reheap() { // SetBaseFee updates the base fee and triggers a re-heap. Note that Removed is not // necessary to call right before SetBaseFee when processing a new block. -func (l *txPricedList) SetBaseFee(baseFee *big.Int) { +func (l *txPricedList) SetBaseFee(baseFee *uint256.Int) { + l.urgent.baseFeeMu.Lock() l.urgent.baseFee = baseFee + l.urgent.baseFeeMu.Unlock() + l.Reheap() } diff --git a/core/tx_list_test.go b/core/tx_list_test.go index ef49cae1dd..80b8c1ef32 100644 --- a/core/tx_list_test.go +++ b/core/tx_list_test.go @@ -17,10 +17,11 @@ package core import ( - "math/big" "math/rand" "testing" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" ) @@ -59,11 +60,15 @@ func BenchmarkTxListAdd(b *testing.B) { for i := 0; i < len(txs); i++ { txs[i] = transaction(uint64(i), 0, key) } + // Insert the transactions in a random order - priceLimit := big.NewInt(int64(DefaultTxPoolConfig.PriceLimit)) + priceLimit := uint256.NewInt(DefaultTxPoolConfig.PriceLimit) b.ResetTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { list := newTxList(true) + for _, v := range rand.Perm(len(txs)) { list.Add(txs[v], DefaultTxPoolConfig.PriceBump) list.Filter(priceLimit, DefaultTxPoolConfig.PriceBump) diff --git a/core/tx_pool.go b/core/tx_pool.go index 7648668688..e98fd2e0ae 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -17,6 +17,7 @@ package core import ( + "context" "errors" "math" "math/big" @@ -25,8 +26,12 @@ import ( "sync/atomic" "time" + "github.com/holiman/uint256" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/common/tracing" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -126,6 +131,11 @@ var ( localGauge = metrics.NewRegisteredGauge("txpool/local", nil) slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) + resetCacheGauge = metrics.NewRegisteredGauge("txpool/resetcache", nil) + reinitCacheGauge = metrics.NewRegisteredGauge("txpool/reinittcache", nil) + hitCacheCounter = metrics.NewRegisteredCounter("txpool/cachehit", nil) + missCacheCounter = metrics.NewRegisteredCounter("txpool/cachemiss", nil) + reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) ) @@ -231,14 +241,17 @@ func (config *TxPoolConfig) sanitize() TxPoolConfig { // current state) and future transactions. Transactions move between those // two states over time as they are received and processed. type TxPool struct { - config TxPoolConfig - chainconfig *params.ChainConfig - chain blockChain - gasPrice *big.Int - txFeed event.Feed - scope event.SubscriptionScope - signer types.Signer - mu sync.RWMutex + config TxPoolConfig + chainconfig *params.ChainConfig + chain blockChain + gasPrice *big.Int + gasPriceUint *uint256.Int + gasPriceMu sync.RWMutex + + txFeed event.Feed + scope event.SubscriptionScope + signer types.Signer + mu sync.RWMutex istanbul bool // Fork indicator whether we are in the istanbul stage. eip2718 bool // Fork indicator whether we are using EIP-2718 type transactions. @@ -251,11 +264,13 @@ type TxPool struct { locals *accountSet // Set of local transaction to exempt from eviction rules journal *txJournal // Journal of local transaction to back up to disk - pending map[common.Address]*txList // All currently processable transactions - queue map[common.Address]*txList // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all *txLookup // All transactions to allow lookups - priced *txPricedList // All transactions sorted by price + pending map[common.Address]*txList // All currently processable transactions + pendingCount int + pendingMu sync.RWMutex + queue map[common.Address]*txList // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + all *txLookup // All transactions to allow lookups + priced *txPricedList // All transactions sorted by price chainHeadCh chan ChainHeadEvent chainHeadSub event.Subscription @@ -300,6 +315,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block reorgShutdownCh: make(chan struct{}), initDoneCh: make(chan struct{}), gasPrice: new(big.Int).SetUint64(config.PriceLimit), + gasPriceUint: uint256.NewInt(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) @@ -376,9 +392,7 @@ func (pool *TxPool) loop() { // Handle stats reporting ticks case <-report.C: - pool.mu.RLock() pending, queued := pool.stats() - pool.mu.RUnlock() stales := int(atomic.LoadInt64(&pool.priced.stales)) if pending != prevPending || queued != prevQueued || stales != prevStales { @@ -388,22 +402,45 @@ func (pool *TxPool) loop() { // Handle inactive account transaction eviction case <-evict.C: - pool.mu.Lock() + now := time.Now() + + var ( + list types.Transactions + tx *types.Transaction + toRemove []common.Hash + ) + + pool.mu.RLock() for addr := range pool.queue { // Skip local transactions from the eviction mechanism if pool.locals.contains(addr) { continue } + // Any non-locals old enough should be removed - if time.Since(pool.beats[addr]) > pool.config.Lifetime { - list := pool.queue[addr].Flatten() - for _, tx := range list { - pool.removeTx(tx.Hash(), true) + if now.Sub(pool.beats[addr]) > pool.config.Lifetime { + list = pool.queue[addr].Flatten() + for _, tx = range list { + toRemove = append(toRemove, tx.Hash()) } + queuedEvictionMeter.Mark(int64(len(list))) } } - pool.mu.Unlock() + + pool.mu.RUnlock() + + if len(toRemove) > 0 { + pool.mu.Lock() + + var hash common.Hash + + for _, hash = range toRemove { + pool.removeTx(hash, true) + } + + pool.mu.Unlock() + } // Handle local transaction journal rotation case <-journal.C: @@ -441,27 +478,45 @@ func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- NewTxsEvent) event.Subscripti // GasPrice returns the current gas price enforced by the transaction pool. func (pool *TxPool) GasPrice() *big.Int { - pool.mu.RLock() - defer pool.mu.RUnlock() + pool.gasPriceMu.RLock() + defer pool.gasPriceMu.RUnlock() return new(big.Int).Set(pool.gasPrice) } +func (pool *TxPool) GasPriceUint256() *uint256.Int { + pool.gasPriceMu.RLock() + defer pool.gasPriceMu.RUnlock() + + return pool.gasPriceUint.Clone() +} + // SetGasPrice updates the minimum price required by the transaction pool for a // new transaction, and drops all transactions below this threshold. func (pool *TxPool) SetGasPrice(price *big.Int) { - pool.mu.Lock() - defer pool.mu.Unlock() + pool.gasPriceMu.Lock() + defer pool.gasPriceMu.Unlock() old := pool.gasPrice pool.gasPrice = price + + if pool.gasPriceUint == nil { + pool.gasPriceUint, _ = uint256.FromBig(price) + } else { + pool.gasPriceUint.SetFromBig(price) + } + // if the min miner fee increased, remove transactions below the new threshold if price.Cmp(old) > 0 { + pool.mu.Lock() + defer pool.mu.Unlock() + // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead drop := pool.all.RemotesBelowTip(price) for _, tx := range drop { pool.removeTx(tx.Hash(), false) } + pool.priced.Removed(len(drop)) } @@ -480,9 +535,6 @@ func (pool *TxPool) Nonce(addr common.Address) uint64 { // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (pool *TxPool) Stats() (int, int) { - pool.mu.RLock() - defer pool.mu.RUnlock() - return pool.stats() } @@ -490,47 +542,69 @@ func (pool *TxPool) Stats() (int, int) { // number of queued (non-executable) transactions. func (pool *TxPool) stats() (int, int) { pending := 0 + + pool.pendingMu.RLock() for _, list := range pool.pending { pending += list.Len() } + pool.pendingMu.RUnlock() + + pool.mu.RLock() + queued := 0 for _, list := range pool.queue { queued += list.Len() } + + pool.mu.RUnlock() + return pending, queued } // Content retrieves the data content of the transaction pool, returning all the // pending as well as queued transactions, grouped by account and sorted by nonce. func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - pool.mu.Lock() - defer pool.mu.Unlock() - pending := make(map[common.Address]types.Transactions) + + pool.pendingMu.RLock() for addr, list := range pool.pending { pending[addr] = list.Flatten() } + pool.pendingMu.RUnlock() + queued := make(map[common.Address]types.Transactions) + + pool.mu.RLock() + for addr, list := range pool.queue { queued[addr] = list.Flatten() } + + pool.mu.RUnlock() + return pending, queued } // ContentFrom retrieves the data content of the transaction pool, returning the // pending as well as queued transactions of this address, grouped by nonce. func (pool *TxPool) ContentFrom(addr common.Address) (types.Transactions, types.Transactions) { - pool.mu.RLock() - defer pool.mu.RUnlock() - var pending types.Transactions + + pool.pendingMu.RLock() if list, ok := pool.pending[addr]; ok { pending = list.Flatten() } + pool.pendingMu.RUnlock() + + pool.mu.RLock() + var queued types.Transactions if list, ok := pool.queue[addr]; ok { queued = list.Flatten() } + + pool.mu.RUnlock() + return pending, queued } @@ -541,35 +615,74 @@ func (pool *TxPool) ContentFrom(addr common.Address) (types.Transactions, types. // The enforceTips parameter can be used to do an extra filtering on the pending // transactions and only return those whose **effective** tip is large enough in // the next pending execution environment. -func (pool *TxPool) Pending(enforceTips bool) map[common.Address]types.Transactions { - pool.mu.Lock() - defer pool.mu.Unlock() +// +//nolint:gocognit +func (pool *TxPool) Pending(ctx context.Context, enforceTips bool) map[common.Address]types.Transactions { + pending := make(map[common.Address]types.Transactions, 10) - pending := make(map[common.Address]types.Transactions) - for addr, list := range pool.pending { - txs := list.Flatten() + tracing.Exec(ctx, "TxpoolPending", "txpool.Pending()", func(ctx context.Context, span trace.Span) { + tracing.ElapsedTime(ctx, span, "txpool.Pending.RLock()", func(ctx context.Context, s trace.Span) { + pool.pendingMu.RLock() + }) - // If the miner requests tip enforcement, cap the lists now - if enforceTips && !pool.locals.contains(addr) { - for i, tx := range txs { - if tx.EffectiveGasTipIntCmp(pool.gasPrice, pool.priced.urgent.baseFee) < 0 { - txs = txs[:i] - break + defer pool.pendingMu.RUnlock() + + pendingAccounts := len(pool.pending) + + var pendingTxs int + + tracing.ElapsedTime(ctx, span, "Loop", func(ctx context.Context, s trace.Span) { + gasPriceUint := uint256.NewInt(0) + baseFee := uint256.NewInt(0) + + for addr, list := range pool.pending { + txs := list.Flatten() + + // If the miner requests tip enforcement, cap the lists now + if enforceTips && !pool.locals.contains(addr) { + for i, tx := range txs { + pool.pendingMu.RUnlock() + + pool.gasPriceMu.RLock() + if pool.gasPriceUint != nil { + gasPriceUint.Set(pool.gasPriceUint) + } + + pool.priced.urgent.baseFeeMu.Lock() + if pool.priced.urgent.baseFee != nil { + baseFee.Set(pool.priced.urgent.baseFee) + } + pool.priced.urgent.baseFeeMu.Unlock() + + pool.gasPriceMu.RUnlock() + + pool.pendingMu.RLock() + + if tx.EffectiveGasTipUintLt(gasPriceUint, baseFee) { + txs = txs[:i] + break + } + } + } + + if len(txs) > 0 { + pending[addr] = txs + pendingTxs += len(txs) } } - } - if len(txs) > 0 { - pending[addr] = txs - } - } + + tracing.SetAttributes(span, + attribute.Int("pending-transactions", pendingTxs), + attribute.Int("pending-accounts", pendingAccounts), + ) + }) + }) + return pending } // Locals retrieves the accounts currently considered local by the pool. func (pool *TxPool) Locals() []common.Address { - pool.mu.Lock() - defer pool.mu.Unlock() - return pool.locals.flatten() } @@ -578,14 +691,22 @@ func (pool *TxPool) Locals() []common.Address { // freely modified by calling code. func (pool *TxPool) local() map[common.Address]types.Transactions { txs := make(map[common.Address]types.Transactions) + + pool.locals.m.RLock() + defer pool.locals.m.RUnlock() + for addr := range pool.locals.accounts { + pool.pendingMu.RLock() if pending := pool.pending[addr]; pending != nil { txs[addr] = append(txs[addr], pending.Flatten()...) } + pool.pendingMu.RUnlock() + if queued := pool.queue[addr]; queued != nil { txs[addr] = append(txs[addr], queued.Flatten()...) } } + return txs } @@ -596,60 +717,84 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { if !pool.eip2718 && tx.Type() != types.LegacyTxType { return ErrTxTypeNotSupported } + // Reject dynamic fee transactions until EIP-1559 activates. if !pool.eip1559 && tx.Type() == types.DynamicFeeTxType { return ErrTxTypeNotSupported } + // Reject transactions over defined size to prevent DOS attacks if uint64(tx.Size()) > txMaxSize { return ErrOversizedData } + // Transactions can't be negative. This may never happen using RLP decoded // transactions but may occur if you create a transaction using the RPC. if tx.Value().Sign() < 0 { return ErrNegativeValue } + // Ensure the transaction doesn't exceed the current block limit gas. if pool.currentMaxGas < tx.Gas() { return ErrGasLimit } + // Sanity check for extremely large numbers - if tx.GasFeeCap().BitLen() > 256 { + gasFeeCap := tx.GasFeeCapRef() + if gasFeeCap.BitLen() > 256 { return ErrFeeCapVeryHigh } - if tx.GasTipCap().BitLen() > 256 { + + // do NOT use uint256 here. results vs *big.Int are different + gasTipCap := tx.GasTipCapRef() + if gasTipCap.BitLen() > 256 { return ErrTipVeryHigh } + // Ensure gasFeeCap is greater than or equal to gasTipCap. - if tx.GasFeeCapIntCmp(tx.GasTipCap()) < 0 { + gasTipCapU, _ := uint256.FromBig(gasTipCap) + if tx.GasFeeCapUIntLt(gasTipCapU) { return ErrTipAboveFeeCap } + // Make sure the transaction is signed properly. from, err := types.Sender(pool.signer, tx) if err != nil { return ErrInvalidSender } + // Drop non-local transactions under our own minimal accepted gas price or tip - if !local && tx.GasTipCapIntCmp(pool.gasPrice) < 0 { + pool.gasPriceMu.RLock() + + if !local && tx.GasTipCapUIntLt(pool.gasPriceUint) { + pool.gasPriceMu.RUnlock() + return ErrUnderpriced } + + pool.gasPriceMu.RUnlock() + // Ensure the transaction adheres to nonce ordering if pool.currentState.GetNonce(from) > tx.Nonce() { return ErrNonceTooLow } + // Transactor should have enough funds to cover the costs // cost == V + GP * GL if pool.currentState.GetBalance(from).Cmp(tx.Cost()) < 0 { return ErrInsufficientFunds } + // Ensure the transaction has more gas than the basic tx fee. intrGas, err := IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, true, pool.istanbul) if err != nil { return err } + if tx.Gas() < intrGas { return ErrIntrinsicGas } + return nil } @@ -682,7 +827,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e if uint64(pool.all.Slots()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it if !isLocal && pool.priced.Underpriced(tx) { - log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) + log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCapUint(), "gasFeeCap", tx.GasFeeCapUint()) underpricedTxMeter.Mark(1) return false, ErrUnderpriced } @@ -710,26 +855,36 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e pool.changesSinceReorg += len(drop) // Kick out the underpriced remote transactions. for _, tx := range drop { - log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) + log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCapUint(), "gasFeeCap", tx.GasFeeCapUint()) underpricedTxMeter.Mark(1) pool.removeTx(tx.Hash(), false) } } // Try to replace an existing transaction in the pending pool from, _ := types.Sender(pool.signer, tx) // already validated - if list := pool.pending[from]; list != nil && list.Overlaps(tx) { + + pool.pendingMu.RLock() + + list := pool.pending[from] + + if list != nil && list.Overlaps(tx) { // Nonce already pending, check if required price bump is met inserted, old := list.Add(tx, pool.config.PriceBump) + pool.pendingCount++ + pool.pendingMu.RUnlock() + if !inserted { pendingDiscardMeter.Mark(1) return false, ErrReplaceUnderpriced } + // New transaction is better, replace old one if old != nil { pool.all.Remove(old.Hash()) pool.priced.Removed(1) pendingReplaceMeter.Mark(1) } + pool.all.Add(tx, isLocal) pool.priced.Put(tx, isLocal) pool.journalTx(from, tx) @@ -738,8 +893,13 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e // Successful promotion, bump the heartbeat pool.beats[from] = time.Now() + return old != nil, nil } + + // it is not an unlocking of unlocked because of the return in previous 'if' + pool.pendingMu.RUnlock() + // New transaction isn't replacing a pending one, push into queue replaced, err = pool.enqueueTx(hash, tx, isLocal, true) if err != nil { @@ -829,19 +989,25 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T }() // Try to insert the transaction into the pending queue + pool.pendingMu.Lock() if pool.pending[addr] == nil { pool.pending[addr] = newTxList(true) } list := pool.pending[addr] inserted, old := list.Add(tx, pool.config.PriceBump) + pool.pendingCount++ + pool.pendingMu.Unlock() + if !inserted { // An older transaction was better, discard this pool.all.Remove(hash) pool.priced.Removed(1) pendingDiscardMeter.Mark(1) + return false } + // Otherwise discard any previous transaction and mark this if old != nil { pool.all.Remove(old.Hash()) @@ -851,11 +1017,13 @@ func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.T // Nothing was replaced, bump the pending counter pendingGauge.Inc(1) } + // Set the potentially new pending nonce and notify any subsystems of the new tx pool.pendingNonces.set(addr, tx.Nonce()+1) // Successful promotion, bump the heartbeat pool.beats[addr] = time.Now() + return true } @@ -871,8 +1039,7 @@ func (pool *TxPool) AddLocals(txs []*types.Transaction) []error { // AddLocal enqueues a single local transaction into the pool if it is valid. This is // a convenience wrapper aroundd AddLocals. func (pool *TxPool) AddLocal(tx *types.Transaction) error { - errs := pool.AddLocals([]*types.Transaction{tx}) - return errs[0] + return pool.addTx(tx, !pool.config.NoLocals, true) } // AddRemotes enqueues a batch of transactions into the pool if they are valid. If the @@ -889,108 +1056,216 @@ func (pool *TxPool) AddRemotesSync(txs []*types.Transaction) []error { return pool.addTxs(txs, false, true) } +func (pool *TxPool) AddRemoteSync(txs *types.Transaction) error { + return pool.addTx(txs, false, true) +} + // This is like AddRemotes with a single transaction, but waits for pool reorganization. Tests use this method. func (pool *TxPool) addRemoteSync(tx *types.Transaction) error { - errs := pool.AddRemotesSync([]*types.Transaction{tx}) - return errs[0] + return pool.AddRemoteSync(tx) } // AddRemote enqueues a single transaction into the pool if it is valid. This is a convenience // wrapper around AddRemotes. -// -// Deprecated: use AddRemotes func (pool *TxPool) AddRemote(tx *types.Transaction) error { - errs := pool.AddRemotes([]*types.Transaction{tx}) - return errs[0] + return pool.addTx(tx, false, false) } // addTxs attempts to queue a batch of transactions if they are valid. func (pool *TxPool) addTxs(txs []*types.Transaction, local, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( - errs = make([]error, len(txs)) + errs []error news = make([]*types.Transaction, 0, len(txs)) + err error + + hash common.Hash ) - for i, tx := range txs { + + for _, tx := range txs { // If the transaction is known, pre-set the error slot - if pool.all.Get(tx.Hash()) != nil { - errs[i] = ErrAlreadyKnown + hash = tx.Hash() + + if pool.all.Get(hash) != nil { + errs = append(errs, ErrAlreadyKnown) knownTxMeter.Mark(1) + continue } + // Exclude transactions with invalid signatures as soon as // possible and cache senders in transactions before // obtaining lock - _, err := types.Sender(pool.signer, tx) + _, err = types.Sender(pool.signer, tx) if err != nil { - errs[i] = ErrInvalidSender + errs = append(errs, ErrInvalidSender) invalidTxMeter.Mark(1) + continue } + // Accumulate all unknown transactions for deeper processing news = append(news, tx) } + if len(news) == 0 { return errs } // Process all the new transaction and merge any errors into the original slice pool.mu.Lock() - newErrs, dirtyAddrs := pool.addTxsLocked(news, local) + errs, dirtyAddrs := pool.addTxsLocked(news, local) pool.mu.Unlock() - var nilSlot = 0 - for _, err := range newErrs { - for errs[nilSlot] != nil { - nilSlot++ - } - errs[nilSlot] = err - nilSlot++ - } // Reorg the pool internals if needed and return done := pool.requestPromoteExecutables(dirtyAddrs) if sync { <-done } + return errs } +// addTxs attempts to queue a batch of transactions if they are valid. +func (pool *TxPool) addTx(tx *types.Transaction, local, sync bool) error { + // Filter out known ones without obtaining the pool lock or recovering signatures + var ( + err error + hash common.Hash + ) + + func() { + // If the transaction is known, pre-set the error slot + hash = tx.Hash() + + if pool.all.Get(hash) != nil { + err = ErrAlreadyKnown + + knownTxMeter.Mark(1) + + return + } + + // Exclude transactions with invalid signatures as soon as + // possible and cache senders in transactions before + // obtaining lock + _, err = types.Sender(pool.signer, tx) + if err != nil { + invalidTxMeter.Mark(1) + + return + } + }() + + if err != nil { + return err + } + + var dirtyAddrs *accountSet + + // Process all the new transaction and merge any errors into the original slice + pool.mu.Lock() + err, dirtyAddrs = pool.addTxLocked(tx, local) + pool.mu.Unlock() + + // Reorg the pool internals if needed and return + done := pool.requestPromoteExecutables(dirtyAddrs) + if sync { + <-done + } + + return err +} + // addTxsLocked attempts to queue a batch of transactions if they are valid. // The transaction pool lock must be held. func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) ([]error, *accountSet) { dirty := newAccountSet(pool.signer) - errs := make([]error, len(txs)) - for i, tx := range txs { - replaced, err := pool.add(tx, local) - errs[i] = err + + var ( + replaced bool + errs []error + ) + + for _, tx := range txs { + var err error + + replaced, err = pool.add(tx, local) if err == nil && !replaced { dirty.addTx(tx) } + + if err != nil { + errs = append(errs, err) + } } + validTxMeter.Mark(int64(len(dirty.accounts))) + return errs, dirty } +func (pool *TxPool) addTxLocked(tx *types.Transaction, local bool) (error, *accountSet) { + dirty := newAccountSet(pool.signer) + + var ( + replaced bool + err error + ) + + replaced, err = pool.add(tx, local) + if err == nil && !replaced { + dirty.addTx(tx) + } + + validTxMeter.Mark(int64(len(dirty.accounts))) + + return err, dirty +} + // Status returns the status (unknown/pending/queued) of a batch of transactions // identified by their hashes. func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { status := make([]TxStatus, len(hashes)) + + var ( + txList *txList + isPending bool + ) + for i, hash := range hashes { tx := pool.Get(hash) if tx == nil { continue } + from, _ := types.Sender(pool.signer, tx) // already validated - pool.mu.RLock() - if txList := pool.pending[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { + + pool.pendingMu.RLock() + + if txList = pool.pending[from]; txList != nil && txList.txs.Has(tx.Nonce()) { status[i] = TxStatusPending - } else if txList := pool.queue[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { - status[i] = TxStatusQueued + isPending = true + } else { + isPending = false } + + pool.pendingMu.RUnlock() + + if !isPending { + pool.mu.RLock() + + if txList := pool.queue[from]; txList != nil && txList.txs.Has(tx.Nonce()) { + status[i] = TxStatusQueued + } + + pool.mu.RUnlock() + } + // implicit else: the tx may have been included into a block between // checking pool.Get and obtaining the lock. In that case, TxStatusUnknown is correct - pool.mu.RUnlock() } + return status } @@ -1013,6 +1288,7 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { if tx == nil { return } + addr, _ := types.Sender(pool.signer, tx) // already validated during insertion // Remove it from the list of known transactions @@ -1020,34 +1296,52 @@ func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { if outofbound { pool.priced.Removed(1) } + if pool.locals.contains(addr) { localGauge.Dec(1) } + // Remove the transaction from the pending lists and reset the account nonce + pool.pendingMu.Lock() + if pending := pool.pending[addr]; pending != nil { if removed, invalids := pending.Remove(tx); removed { + pool.pendingCount-- + // If no more pending transactions are left, remove the list if pending.Empty() { delete(pool.pending, addr) } + + pool.pendingMu.Unlock() + // Postpone any invalidated transactions for _, tx := range invalids { // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(tx.Hash(), tx, false, false) } + // Update the account nonce if needed pool.pendingNonces.setIfLower(addr, tx.Nonce()) + // Reduce the pending counter pendingGauge.Dec(int64(1 + len(invalids))) + return } + + pool.pendingMu.TryLock() } + + pool.pendingMu.Unlock() + // Transaction is in the future queue if future := pool.queue[addr]; future != nil { if removed, _ := future.Remove(tx); removed { // Reduce the queued counter queuedGauge.Dec(1) } + if future.Empty() { delete(pool.queue, addr) delete(pool.beats, addr) @@ -1103,8 +1397,10 @@ func (pool *TxPool) scheduleReorgLoop() { for { // Launch next background reorg if needed if curDone == nil && launchNextRun { + ctx := context.Background() + // Run the background reorg and announcements - go pool.runReorg(nextDone, reset, dirtyAccounts, queuedEvents) + go pool.runReorg(ctx, nextDone, reset, dirtyAccounts, queuedEvents) // Prepare everything for the next round of reorg curDone, nextDone = nextDone, make(chan struct{}) @@ -1159,86 +1455,175 @@ func (pool *TxPool) scheduleReorgLoop() { } // runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. -func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*txSortedMap) { - defer func(t0 time.Time) { - reorgDurationTimer.Update(time.Since(t0)) - }(time.Now()) - defer close(done) +// +//nolint:gocognit +func (pool *TxPool) runReorg(ctx context.Context, done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*txSortedMap) { + tracing.Exec(ctx, "TxPoolReorg", "txpool-reorg", func(ctx context.Context, span trace.Span) { + defer func(t0 time.Time) { + reorgDurationTimer.Update(time.Since(t0)) + }(time.Now()) - var promoteAddrs []common.Address - if dirtyAccounts != nil && reset == nil { - // Only dirty accounts need to be promoted, unless we're resetting. - // For resets, all addresses in the tx queue will be promoted and - // the flatten operation can be avoided. - promoteAddrs = dirtyAccounts.flatten() - } - pool.mu.Lock() - if reset != nil { - // Reset from the old head to the new, rescheduling any reorged transactions - pool.reset(reset.oldHead, reset.newHead) + defer close(done) - // Nonces were reset, discard any events that became stale - for addr := range events { - events[addr].Forward(pool.pendingNonces.get(addr)) - if events[addr].Len() == 0 { - delete(events, addr) + var promoteAddrs []common.Address + + tracing.ElapsedTime(ctx, span, "01 dirty accounts flattening", func(_ context.Context, innerSpan trace.Span) { + if dirtyAccounts != nil && reset == nil { + // Only dirty accounts need to be promoted, unless we're resetting. + // For resets, all addresses in the tx queue will be promoted and + // the flatten operation can be avoided. + promoteAddrs = dirtyAccounts.flatten() } - } - // Reset needs promote for all addresses - promoteAddrs = make([]common.Address, 0, len(pool.queue)) - for addr := range pool.queue { - promoteAddrs = append(promoteAddrs, addr) - } - } - // Check for pending transactions for every account that sent new ones - promoted := pool.promoteExecutables(promoteAddrs) - // If a new block appeared, validate the pool of pending transactions. This will - // remove any transaction that has been included in the block or was invalidated - // because of another transaction (e.g. higher gas price). - if reset != nil { - pool.demoteUnexecutables() - if reset.newHead != nil { - if pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { - // london fork enabled, reset given the base fee - pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) - pool.priced.SetBaseFee(pendingBaseFee) - } else { - // london fork not enabled, reheap to "reset" the priced list - pool.priced.Reheap() + tracing.SetAttributes( + innerSpan, + attribute.Int("promoteAddresses-flatten", len(promoteAddrs)), + ) + }) + + tracing.ElapsedTime(ctx, span, "02 obtaining pool.WMutex", func(_ context.Context, _ trace.Span) { + pool.mu.Lock() + }) + + if reset != nil { + tracing.ElapsedTime(ctx, span, "03 reset-head reorg", func(_ context.Context, innerSpan trace.Span) { + + // Reset from the old head to the new, rescheduling any reorged transactions + tracing.ElapsedTime(ctx, innerSpan, "04 reset-head-itself reorg", func(_ context.Context, innerSpan trace.Span) { + pool.reset(reset.oldHead, reset.newHead) + }) + + tracing.SetAttributes( + innerSpan, + attribute.Int("events-reset-head", len(events)), + ) + + // Nonces were reset, discard any events that became stale + for addr := range events { + events[addr].Forward(pool.pendingNonces.get(addr)) + + if events[addr].Len() == 0 { + delete(events, addr) + } + } + + // Reset needs promote for all addresses + promoteAddrs = make([]common.Address, 0, len(pool.queue)) + for addr := range pool.queue { + promoteAddrs = append(promoteAddrs, addr) + } + + tracing.SetAttributes( + innerSpan, + attribute.Int("promoteAddresses-reset-head", len(promoteAddrs)), + ) + }) + } + + // Check for pending transactions for every account that sent new ones + var promoted []*types.Transaction + + tracing.ElapsedTime(ctx, span, "05 promoteExecutables", func(_ context.Context, _ trace.Span) { + promoted = pool.promoteExecutables(promoteAddrs) + }) + + tracing.SetAttributes( + span, + attribute.Int("count.promoteAddresses-reset-head", len(promoteAddrs)), + attribute.Int("count.all", pool.all.Count()), + attribute.Int("count.pending", len(pool.pending)), + attribute.Int("count.queue", len(pool.queue)), + ) + + // If a new block appeared, validate the pool of pending transactions. This will + // remove any transaction that has been included in the block or was invalidated + // because of another transaction (e.g. higher gas price). + + if reset != nil { + tracing.ElapsedTime(ctx, span, "new block", func(_ context.Context, innerSpan trace.Span) { + + tracing.ElapsedTime(ctx, innerSpan, "06 demoteUnexecutables", func(_ context.Context, _ trace.Span) { + pool.demoteUnexecutables() + }) + + var nonces map[common.Address]uint64 + + tracing.ElapsedTime(ctx, innerSpan, "07 set_base_fee", func(_ context.Context, _ trace.Span) { + if reset.newHead != nil { + if pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { + // london fork enabled, reset given the base fee + pendingBaseFee := misc.CalcBaseFeeUint(pool.chainconfig, reset.newHead) + pool.priced.SetBaseFee(pendingBaseFee) + } else { + // london fork not enabled, reheap to "reset" the priced list + pool.priced.Reheap() + } + } + + // Update all accounts to the latest known pending nonce + nonces = make(map[common.Address]uint64, len(pool.pending)) + }) + + tracing.ElapsedTime(ctx, innerSpan, "08 obtaining pendingMu.RMutex", func(_ context.Context, _ trace.Span) { + pool.pendingMu.RLock() + }) + + var highestPending *types.Transaction + + tracing.ElapsedTime(ctx, innerSpan, "09 fill nonces", func(_ context.Context, innerSpan trace.Span) { + for addr, list := range pool.pending { + highestPending = list.LastElement() + nonces[addr] = highestPending.Nonce() + 1 + } + }) + + pool.pendingMu.RUnlock() + + tracing.ElapsedTime(ctx, innerSpan, "10 reset nonces", func(_ context.Context, _ trace.Span) { + pool.pendingNonces.setAll(nonces) + }) + }) + } + + // Ensure pool.queue and pool.pending sizes stay within the configured limits. + tracing.ElapsedTime(ctx, span, "11 truncatePending", func(_ context.Context, _ trace.Span) { + pool.truncatePending() + }) + + tracing.ElapsedTime(ctx, span, "12 truncateQueue", func(_ context.Context, _ trace.Span) { + pool.truncateQueue() + }) + + dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) + pool.changesSinceReorg = 0 // Reset change counter + + pool.mu.Unlock() + + // Notify subsystems for newly added transactions + tracing.ElapsedTime(ctx, span, "13 notify about new transactions", func(_ context.Context, _ trace.Span) { + for _, tx := range promoted { + addr, _ := types.Sender(pool.signer, tx) + + if _, ok := events[addr]; !ok { + events[addr] = newTxSortedMap() + } + + events[addr].Put(tx) } - } - // Update all accounts to the latest known pending nonce - nonces := make(map[common.Address]uint64, len(pool.pending)) - for addr, list := range pool.pending { - highestPending := list.LastElement() - nonces[addr] = highestPending.Nonce() + 1 - } - pool.pendingNonces.setAll(nonces) - } - // Ensure pool.queue and pool.pending sizes stay within the configured limits. - pool.truncatePending() - pool.truncateQueue() + }) - dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) - pool.changesSinceReorg = 0 // Reset change counter - pool.mu.Unlock() + if len(events) > 0 { + tracing.ElapsedTime(ctx, span, "14 txFeed", func(_ context.Context, _ trace.Span) { + var txs []*types.Transaction - // Notify subsystems for newly added transactions - for _, tx := range promoted { - addr, _ := types.Sender(pool.signer, tx) - if _, ok := events[addr]; !ok { - events[addr] = newTxSortedMap() + for _, set := range events { + txs = append(txs, set.Flatten()...) + } + + pool.txFeed.Send(NewTxsEvent{txs}) + }) } - events[addr].Put(tx) - } - if len(events) > 0 { - var txs []*types.Transaction - for _, set := range events { - txs = append(txs, set.Flatten()...) - } - pool.txFeed.Send(NewTxsEvent{txs}) - } + }) } // reset retrieves the current state of the blockchain and ensures the content @@ -1337,64 +1722,100 @@ func (pool *TxPool) reset(oldHead, newHead *types.Header) { // invalidated transactions (low nonce, low balance) are deleted. func (pool *TxPool) promoteExecutables(accounts []common.Address) []*types.Transaction { // Track the promoted transactions to broadcast them at once - var promoted []*types.Transaction + var ( + promoted []*types.Transaction + promotedLen int + forwards types.Transactions + forwardsLen int + caps types.Transactions + capsLen int + drops types.Transactions + dropsLen int + list *txList + hash common.Hash + readies types.Transactions + readiesLen int + ) + + balance := uint256.NewInt(0) // Iterate over all accounts and promote any executable transactions for _, addr := range accounts { - list := pool.queue[addr] + list = pool.queue[addr] if list == nil { continue // Just in case someone calls with a non existing account } + // Drop all transactions that are deemed too old (low nonce) - forwards := list.Forward(pool.currentState.GetNonce(addr)) + forwards = list.Forward(pool.currentState.GetNonce(addr)) + forwardsLen = len(forwards) + for _, tx := range forwards { - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) } - log.Trace("Removed old queued transactions", "count", len(forwards)) + + log.Trace("Removed old queued transactions", "count", forwardsLen) + // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) + balance.SetFromBig(pool.currentState.GetBalance(addr)) + + drops, _ = list.Filter(balance, pool.currentMaxGas) + dropsLen = len(drops) + for _, tx := range drops { - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) } - log.Trace("Removed unpayable queued transactions", "count", len(drops)) - queuedNofundsMeter.Mark(int64(len(drops))) + + log.Trace("Removed unpayable queued transactions", "count", dropsLen) + queuedNofundsMeter.Mark(int64(dropsLen)) // Gather all executable transactions and promote them - readies := list.Ready(pool.pendingNonces.get(addr)) + readies = list.Ready(pool.pendingNonces.get(addr)) + readiesLen = len(readies) + for _, tx := range readies { - hash := tx.Hash() + hash = tx.Hash() if pool.promoteTx(addr, hash, tx) { promoted = append(promoted, tx) } } - log.Trace("Promoted queued transactions", "count", len(promoted)) - queuedGauge.Dec(int64(len(readies))) + + log.Trace("Promoted queued transactions", "count", promotedLen) + queuedGauge.Dec(int64(readiesLen)) // Drop all transactions over the allowed limit - var caps types.Transactions if !pool.locals.contains(addr) { caps = list.Cap(int(pool.config.AccountQueue)) + capsLen = len(caps) + for _, tx := range caps { - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) + log.Trace("Removed cap-exceeding queued transaction", "hash", hash) } - queuedRateLimitMeter.Mark(int64(len(caps))) + + queuedRateLimitMeter.Mark(int64(capsLen)) } + // Mark all the items dropped as removed - pool.priced.Removed(len(forwards) + len(drops) + len(caps)) - queuedGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) + pool.priced.Removed(forwardsLen + dropsLen + capsLen) + + queuedGauge.Dec(int64(forwardsLen + dropsLen + capsLen)) + if pool.locals.contains(addr) { - localGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) + localGauge.Dec(int64(forwardsLen + dropsLen + capsLen)) } + // Delete the entire queue entry if it became empty. if list.Empty() { delete(pool.queue, addr) delete(pool.beats, addr) } } + return promoted } @@ -1402,86 +1823,162 @@ func (pool *TxPool) promoteExecutables(accounts []common.Address) []*types.Trans // pending limit. The algorithm tries to reduce transaction counts by an approximately // equal number for all for accounts with many pending transactions. func (pool *TxPool) truncatePending() { - pending := uint64(0) - for _, list := range pool.pending { - pending += uint64(list.Len()) - } + pending := uint64(pool.pendingCount) if pending <= pool.config.GlobalSlots { return } pendingBeforeCap := pending + + var listLen int + + type pair struct { + address common.Address + value int64 + } + // Assemble a spam order to penalize large transactors first - spammers := prque.New(nil) + spammers := make([]pair, 0, 8) + count := 0 + + var ok bool + + pool.pendingMu.RLock() for addr, list := range pool.pending { // Only evict transactions from high rollers - if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { - spammers.Push(addr, int64(list.Len())) + listLen = len(list.txs.items) + + pool.pendingMu.RUnlock() + + pool.locals.m.RLock() + + if uint64(listLen) > pool.config.AccountSlots { + if _, ok = pool.locals.accounts[addr]; ok { + pool.locals.m.RUnlock() + + pool.pendingMu.RLock() + + continue + } + + count++ + + spammers = append(spammers, pair{addr, int64(listLen)}) } + + pool.locals.m.RUnlock() + + pool.pendingMu.RLock() } + + pool.pendingMu.RUnlock() + // Gradually drop transactions from offenders - offenders := []common.Address{} - for pending > pool.config.GlobalSlots && !spammers.Empty() { + offenders := make([]common.Address, 0, len(spammers)) + sort.Slice(spammers, func(i, j int) bool { + return spammers[i].value < spammers[j].value + }) + + var ( + offender common.Address + caps types.Transactions + capsLen int + list *txList + hash common.Hash + ) + + // todo: metrics: spammers, offenders, total loops + for len(spammers) != 0 && pending > pool.config.GlobalSlots { // Retrieve the next offender if not local address - offender, _ := spammers.Pop() - offenders = append(offenders, offender.(common.Address)) + offender, spammers = spammers[len(spammers)-1].address, spammers[:len(spammers)-1] + offenders = append(offenders, offender) + + var threshold int // Equalize balances until all the same or below threshold if len(offenders) > 1 { // Calculate the equalization threshold for all current offenders - threshold := pool.pending[offender.(common.Address)].Len() + pool.pendingMu.RLock() + threshold = len(pool.pending[offender].txs.items) // Iteratively reduce all offenders until below limit or threshold reached for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { for i := 0; i < len(offenders)-1; i++ { - list := pool.pending[offenders[i]] + list = pool.pending[offenders[i]] + + caps = list.Cap(len(list.txs.items) - 1) + capsLen = len(caps) + + pool.pendingMu.RUnlock() - caps := list.Cap(list.Len() - 1) for _, tx := range caps { // Drop the transaction from the global pools too - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) // Update the account nonce to the dropped transaction pool.pendingNonces.setIfLower(offenders[i], tx.Nonce()) log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) } - pool.priced.Removed(len(caps)) - pendingGauge.Dec(int64(len(caps))) + + pool.priced.Removed(capsLen) + + pendingGauge.Dec(int64(capsLen)) if pool.locals.contains(offenders[i]) { - localGauge.Dec(int64(len(caps))) + localGauge.Dec(int64(capsLen)) } + pending-- + + pool.pendingMu.RLock() } } + + pool.pendingMu.RUnlock() } } // If still above threshold, reduce to limit or min allowance if pending > pool.config.GlobalSlots && len(offenders) > 0 { + + pool.pendingMu.RLock() + for pending > pool.config.GlobalSlots && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > pool.config.AccountSlots { for _, addr := range offenders { - list := pool.pending[addr] + list = pool.pending[addr] + + caps = list.Cap(len(list.txs.items) - 1) + capsLen = len(caps) + + pool.pendingMu.RUnlock() - caps := list.Cap(list.Len() - 1) for _, tx := range caps { // Drop the transaction from the global pools too - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) // Update the account nonce to the dropped transaction pool.pendingNonces.setIfLower(addr, tx.Nonce()) log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) } - pool.priced.Removed(len(caps)) - pendingGauge.Dec(int64(len(caps))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(caps))) + + pool.priced.Removed(capsLen) + + pendingGauge.Dec(int64(capsLen)) + + if _, ok = pool.locals.accounts[addr]; ok { + localGauge.Dec(int64(capsLen)) } + pending-- + + pool.pendingMu.RLock() } } + + pool.pendingMu.RUnlock() } + pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending)) } @@ -1504,27 +2001,52 @@ func (pool *TxPool) truncateQueue() { } sort.Sort(addresses) + var ( + tx *types.Transaction + txs types.Transactions + list *txList + addr addressByHeartbeat + size uint64 + ) + // Drop transactions until the total is below the limit or only locals remain for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { - addr := addresses[len(addresses)-1] - list := pool.queue[addr.address] + addr = addresses[len(addresses)-1] + list = pool.queue[addr.address] addresses = addresses[:len(addresses)-1] + var ( + listFlatten types.Transactions + isSet bool + ) + // Drop all transactions if they are less than the overflow - if size := uint64(list.Len()); size <= drop { - for _, tx := range list.Flatten() { + if size = uint64(list.Len()); size <= drop { + listFlatten = list.Flatten() + isSet = true + + for _, tx = range listFlatten { pool.removeTx(tx.Hash(), true) } + drop -= size queuedRateLimitMeter.Mark(int64(size)) + continue } + // Otherwise drop only last few transactions - txs := list.Flatten() + if !isSet { + listFlatten = list.Flatten() + } + + txs = listFlatten for i := len(txs) - 1; i >= 0 && drop > 0; i-- { pool.removeTx(txs[i].Hash(), true) + drop-- + queuedRateLimitMeter.Mark(1) } } @@ -1538,56 +2060,98 @@ func (pool *TxPool) truncateQueue() { // is always explicitly triggered by SetBaseFee and it would be unnecessary and wasteful // to trigger a re-heap is this function func (pool *TxPool) demoteUnexecutables() { + balance := uint256.NewInt(0) + + var ( + olds types.Transactions + oldsLen int + hash common.Hash + drops types.Transactions + dropsLen int + invalids types.Transactions + invalidsLen int + gapped types.Transactions + gappedLen int + ) + // Iterate over all accounts and demote any non-executable transactions + pool.pendingMu.RLock() + for addr, list := range pool.pending { nonce := pool.currentState.GetNonce(addr) // Drop all transactions that are deemed too old (low nonce) - olds := list.Forward(nonce) + olds = list.Forward(nonce) + oldsLen = len(olds) + for _, tx := range olds { - hash := tx.Hash() + hash = tx.Hash() pool.all.Remove(hash) log.Trace("Removed old pending transaction", "hash", hash) } + // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(pool.currentState.GetBalance(addr), pool.currentMaxGas) + balance.SetFromBig(pool.currentState.GetBalance(addr)) + drops, invalids = list.Filter(balance, pool.currentMaxGas) + dropsLen = len(drops) + invalidsLen = len(invalids) + for _, tx := range drops { - hash := tx.Hash() + hash = tx.Hash() + log.Trace("Removed unpayable pending transaction", "hash", hash) + pool.all.Remove(hash) } - pendingNofundsMeter.Mark(int64(len(drops))) + + pendingNofundsMeter.Mark(int64(dropsLen)) for _, tx := range invalids { - hash := tx.Hash() + hash = tx.Hash() + log.Trace("Demoting pending transaction", "hash", hash) // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(hash, tx, false, false) } - pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) + + pendingGauge.Dec(int64(oldsLen + dropsLen + invalidsLen)) + if pool.locals.contains(addr) { - localGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) + localGauge.Dec(int64(oldsLen + dropsLen + invalidsLen)) } // If there's a gap in front, alert (should never happen) and postpone all transactions if list.Len() > 0 && list.txs.Get(nonce) == nil { - gapped := list.Cap(0) + gapped = list.Cap(0) + gappedLen = len(gapped) + for _, tx := range gapped { - hash := tx.Hash() + hash = tx.Hash() log.Error("Demoting invalidated transaction", "hash", hash) // Internal shuffle shouldn't touch the lookup set. pool.enqueueTx(hash, tx, false, false) } - pendingGauge.Dec(int64(len(gapped))) + + pendingGauge.Dec(int64(gappedLen)) // This might happen in a reorg, so log it to the metering - blockReorgInvalidatedTx.Mark(int64(len(gapped))) + blockReorgInvalidatedTx.Mark(int64(gappedLen)) } + // Delete the entire pending entry if it became empty. if list.Empty() { + pool.pendingMu.RUnlock() + pool.pendingMu.Lock() + + pool.pendingCount -= pool.pending[addr].Len() delete(pool.pending, addr) + + pool.pendingMu.Unlock() + pool.pendingMu.RLock() } } + + pool.pendingMu.RUnlock() } // addressByHeartbeat is an account address tagged with its last activity timestamp. @@ -1605,9 +2169,10 @@ func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } // accountSet is simply a set of addresses to check for existence, and a signer // capable of deriving addresses from transactions. type accountSet struct { - accounts map[common.Address]struct{} - signer types.Signer - cache *[]common.Address + accounts map[common.Address]struct{} + accountsFlatted []common.Address + signer types.Signer + m sync.RWMutex } // newAccountSet creates a new address set with an associated signer for sender @@ -1625,17 +2190,26 @@ func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet { // contains checks if a given address is contained within the set. func (as *accountSet) contains(addr common.Address) bool { + as.m.RLock() + defer as.m.RUnlock() + _, exist := as.accounts[addr] return exist } func (as *accountSet) empty() bool { + as.m.RLock() + defer as.m.RUnlock() + return len(as.accounts) == 0 } // containsTx checks if the sender of a given tx is within the set. If the sender // cannot be derived, this method returns false. func (as *accountSet) containsTx(tx *types.Transaction) bool { + as.m.RLock() + defer as.m.RUnlock() + if addr, err := types.Sender(as.signer, tx); err == nil { return as.contains(addr) } @@ -1644,8 +2218,14 @@ func (as *accountSet) containsTx(tx *types.Transaction) bool { // add inserts a new address into the set to track. func (as *accountSet) add(addr common.Address) { + as.m.Lock() + defer as.m.Unlock() + + if _, ok := as.accounts[addr]; !ok { + as.accountsFlatted = append(as.accountsFlatted, addr) + } + as.accounts[addr] = struct{}{} - as.cache = nil } // addTx adds the sender of tx into the set. @@ -1658,22 +2238,25 @@ func (as *accountSet) addTx(tx *types.Transaction) { // flatten returns the list of addresses within this set, also caching it for later // reuse. The returned slice should not be changed! func (as *accountSet) flatten() []common.Address { - if as.cache == nil { - accounts := make([]common.Address, 0, len(as.accounts)) - for account := range as.accounts { - accounts = append(accounts, account) - } - as.cache = &accounts - } - return *as.cache + as.m.RLock() + defer as.m.RUnlock() + + return as.accountsFlatted } // merge adds all addresses from the 'other' set into 'as'. func (as *accountSet) merge(other *accountSet) { + var ok bool + + as.m.Lock() + defer as.m.Unlock() + for addr := range other.accounts { + if _, ok = as.accounts[addr]; !ok { + as.accountsFlatted = append(as.accountsFlatted, addr) + } as.accounts[addr] = struct{}{} } - as.cache = nil } // txLookup is used internally by TxPool to track transactions while allowing @@ -1829,7 +2412,10 @@ func (t *txLookup) RemoteToLocals(locals *accountSet) int { var migrated int for hash, tx := range t.remotes { if locals.containsTx(tx) { + locals.m.Lock() t.locals[hash] = tx + locals.m.Unlock() + delete(t.remotes, hash) migrated += 1 } diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 664ca6c9d4..63f712bb9c 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "errors" "fmt" + "io" "io/ioutil" "math/big" "math/rand" @@ -32,11 +33,15 @@ import ( "testing" "time" + "github.com/holiman/uint256" + "go.uber.org/goleak" "gonum.org/v1/gonum/floats" "gonum.org/v1/gonum/stat" "pgregory.net/rapid" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/debug" + "github.com/ethereum/go-ethereum/common/leak" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -98,7 +103,7 @@ func transaction(nonce uint64, gaslimit uint64, key *ecdsa.PrivateKey) *types.Tr } func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ecdsa.PrivateKey) *types.Transaction { - tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{}, big.NewInt(100), gaslimit, gasprice, nil), types.HomesteadSigner{}, key) + tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{0x01}, big.NewInt(100), gaslimit, gasprice, nil), types.HomesteadSigner{}, key) return tx } @@ -153,12 +158,17 @@ func validateTxPoolInternals(pool *TxPool) error { if total := pool.all.Count(); total != pending+queued { return fmt.Errorf("total transaction count %d != %d pending + %d queued", total, pending, queued) } + pool.priced.Reheap() priced, remote := pool.priced.urgent.Len()+pool.priced.floating.Len(), pool.all.RemoteCount() if priced != remote { return fmt.Errorf("total priced transaction count %d != %d", priced, remote) } + // Ensure the next nonce to assign is the correct one + pool.pendingMu.RLock() + defer pool.pendingMu.RUnlock() + for addr, txs := range pool.pending { // Find the last transaction var last uint64 @@ -167,10 +177,12 @@ func validateTxPoolInternals(pool *TxPool) error { last = nonce } } + if nonce := pool.pendingNonces.get(addr); nonce != last+1 { return fmt.Errorf("pending nonce mismatch: have %v, want %v", nonce, last+1) } } + return nil } @@ -325,10 +337,18 @@ func TestInvalidTransactions(t *testing.T) { } tx = transaction(1, 100000, key) + + pool.gasPriceMu.Lock() + pool.gasPrice = big.NewInt(1000) - if err := pool.AddRemote(tx); err != ErrUnderpriced { + pool.gasPriceUint = uint256.NewInt(1000) + + pool.gasPriceMu.Unlock() + + if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { t.Error("expected", ErrUnderpriced, "got", err) } + if err := pool.AddLocal(tx); err != nil { t.Error("expected", nil, "got", err) } @@ -347,9 +367,12 @@ func TestTransactionQueue(t *testing.T) { pool.enqueueTx(tx.Hash(), tx, false, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) + + pool.pendingMu.RLock() if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) } + pool.pendingMu.RUnlock() tx = transaction(1, 100, key) from, _ = deriveSender(tx) @@ -357,9 +380,13 @@ func TestTransactionQueue(t *testing.T) { pool.enqueueTx(tx.Hash(), tx, false, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) + + pool.pendingMu.RLock() if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { t.Error("expected transaction to be in tx pool") } + pool.pendingMu.RUnlock() + if len(pool.queue) > 0 { t.Error("expected transaction queue to be empty. is", len(pool.queue)) } @@ -383,9 +410,13 @@ func TestTransactionQueue2(t *testing.T) { pool.enqueueTx(tx3.Hash(), tx3, false, true) pool.promoteExecutables([]common.Address{from}) + + pool.pendingMu.RLock() if len(pool.pending) != 1 { t.Error("expected pending length to be 1, got", len(pool.pending)) } + pool.pendingMu.RUnlock() + if pool.queue[from].Len() != 2 { t.Error("expected len(queue) == 2, got", pool.queue[from].Len()) } @@ -399,8 +430,10 @@ func TestTransactionNegativeValue(t *testing.T) { tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), 100, big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) + testAddBalance(pool, from, big.NewInt(1)) - if err := pool.AddRemote(tx); err != ErrNegativeValue { + + if err := pool.AddRemote(tx); !errors.Is(err, ErrNegativeValue) { t.Error("expected", ErrNegativeValue, "got", err) } } @@ -413,7 +446,7 @@ func TestTransactionTipAboveFeeCap(t *testing.T) { tx := dynamicFeeTx(0, 100, big.NewInt(1), big.NewInt(2), key) - if err := pool.AddRemote(tx); err != ErrTipAboveFeeCap { + if err := pool.AddRemote(tx); !errors.Is(err, ErrTipAboveFeeCap) { t.Error("expected", ErrTipAboveFeeCap, "got", err) } } @@ -428,12 +461,12 @@ func TestTransactionVeryHighValues(t *testing.T) { veryBigNumber.Lsh(veryBigNumber, 300) tx := dynamicFeeTx(0, 100, big.NewInt(1), veryBigNumber, key) - if err := pool.AddRemote(tx); err != ErrTipVeryHigh { + if err := pool.AddRemote(tx); !errors.Is(err, ErrTipVeryHigh) { t.Error("expected", ErrTipVeryHigh, "got", err) } tx2 := dynamicFeeTx(0, 100, veryBigNumber, big.NewInt(1), key) - if err := pool.AddRemote(tx2); err != ErrFeeCapVeryHigh { + if err := pool.AddRemote(tx2); !errors.Is(err, ErrFeeCapVeryHigh) { t.Error("expected", ErrFeeCapVeryHigh, "got", err) } } @@ -495,23 +528,32 @@ func TestTransactionDoubleNonce(t *testing.T) { if replace, err := pool.add(tx2, false); err != nil || !replace { t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } + <-pool.requestPromoteExecutables(newAccountSet(signer, addr)) + + pool.pendingMu.RLock() if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } if tx := pool.pending[addr].txs.items[0]; tx.Hash() != tx2.Hash() { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } + pool.pendingMu.RUnlock() // Add the third transaction and ensure it's not saved (smaller price) pool.add(tx3, false) + <-pool.requestPromoteExecutables(newAccountSet(signer, addr)) + + pool.pendingMu.RLock() if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) } if tx := pool.pending[addr].txs.items[0]; tx.Hash() != tx2.Hash() { t.Errorf("transaction mismatch: have %x, want %x", tx.Hash(), tx2.Hash()) } + pool.pendingMu.RUnlock() + // Ensure the total transaction count is correct if pool.all.Count() != 1 { t.Error("expected 1 total transactions, got", pool.all.Count()) @@ -530,9 +572,13 @@ func TestTransactionMissingNonce(t *testing.T) { if _, err := pool.add(tx, false); err != nil { t.Error("didn't expect error", err) } + + pool.pendingMu.RLock() if len(pool.pending) != 0 { t.Error("expected 0 pending transactions, got", len(pool.pending)) } + pool.pendingMu.RUnlock() + if pool.queue[addr].Len() != 1 { t.Error("expected 1 queued transaction, got", pool.queue[addr].Len()) } @@ -603,19 +649,27 @@ func TestTransactionDropping(t *testing.T) { pool.enqueueTx(tx12.Hash(), tx12, false, true) // Check that pre and post validations leave the pool as is + pool.pendingMu.RLock() if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } + pool.pendingMu.RUnlock() + if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } if pool.all.Count() != 6 { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 6) } + <-pool.requestReset(nil, nil) + + pool.pendingMu.RLock() if pool.pending[account].Len() != 3 { t.Errorf("pending transaction mismatch: have %d, want %d", pool.pending[account].Len(), 3) } + pool.pendingMu.RUnlock() + if pool.queue[account].Len() != 3 { t.Errorf("queued transaction mismatch: have %d, want %d", pool.queue[account].Len(), 3) } @@ -626,6 +680,7 @@ func TestTransactionDropping(t *testing.T) { testAddBalance(pool, account, big.NewInt(-650)) <-pool.requestReset(nil, nil) + pool.pendingMu.RLock() if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) } @@ -635,6 +690,8 @@ func TestTransactionDropping(t *testing.T) { if _, ok := pool.pending[account].txs.items[tx2.Nonce()]; ok { t.Errorf("out-of-fund pending transaction present: %v", tx1) } + pool.pendingMu.RUnlock() + if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } @@ -651,12 +708,15 @@ func TestTransactionDropping(t *testing.T) { atomic.StoreUint64(&pool.chain.(*testBlockChain).gasLimit, 100) <-pool.requestReset(nil, nil) + pool.pendingMu.RLock() if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { t.Errorf("funded pending transaction missing: %v", tx0) } if _, ok := pool.pending[account].txs.items[tx1.Nonce()]; ok { t.Errorf("over-gased pending transaction present: %v", tx1) } + pool.pendingMu.RUnlock() + if _, ok := pool.queue[account].txs.items[tx10.Nonce()]; !ok { t.Errorf("funded queued transaction missing: %v", tx10) } @@ -711,19 +771,27 @@ func TestTransactionPostponing(t *testing.T) { } } // Check that pre and post validations leave the pool as is + pool.pendingMu.RLock() if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } + pool.pendingMu.RUnlock() + if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } if pool.all.Count() != len(txs) { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), len(txs)) } + <-pool.requestReset(nil, nil) + + pool.pendingMu.RLock() if pending := pool.pending[accs[0]].Len() + pool.pending[accs[1]].Len(); pending != len(txs) { t.Errorf("pending transaction mismatch: have %d, want %d", pending, len(txs)) } + pool.pendingMu.RUnlock() + if len(pool.queue) != 0 { t.Errorf("queued accounts mismatch: have %d, want %d", len(pool.queue), 0) } @@ -738,12 +806,17 @@ func TestTransactionPostponing(t *testing.T) { // The first account's first transaction remains valid, check that subsequent // ones are either filtered out, or queued up for later. + pool.pendingMu.RLock() if _, ok := pool.pending[accs[0]].txs.items[txs[0].Nonce()]; !ok { t.Errorf("tx %d: valid and funded transaction missing from pending pool: %v", 0, txs[0]) } + pool.pendingMu.RUnlock() + if _, ok := pool.queue[accs[0]].txs.items[txs[0].Nonce()]; ok { t.Errorf("tx %d: valid and funded transaction present in future queue: %v", 0, txs[0]) } + + pool.pendingMu.RLock() for i, tx := range txs[1:100] { if i%2 == 1 { if _, ok := pool.pending[accs[0]].txs.items[tx.Nonce()]; ok { @@ -761,11 +834,16 @@ func TestTransactionPostponing(t *testing.T) { } } } + pool.pendingMu.RUnlock() + // The second account's first transaction got invalid, check that all transactions // are either filtered out, or queued up for later. + pool.pendingMu.RLock() if pool.pending[accs[1]] != nil { t.Errorf("invalidated account still has pending transactions") } + pool.pendingMu.RUnlock() + for i, tx := range txs[100:] { if i%2 == 1 { if _, ok := pool.queue[accs[1]].txs.items[tx.Nonce()]; !ok { @@ -854,9 +932,13 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { if err := pool.addRemoteSync(transaction(i, 100000, key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } + + pool.pendingMu.RLock() if len(pool.pending) != 0 { t.Errorf("tx %d: pending pool size mismatch: have %d, want %d", i, len(pool.pending), 0) } + pool.pendingMu.RUnlock() + if i <= testTxPoolConfig.AccountQueue { if pool.queue[account].Len() != int(i) { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), i) @@ -935,6 +1017,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { for i := uint64(0); i < 3*config.GlobalQueue; i++ { txs = append(txs, transaction(i+1, 100000, local)) } + pool.AddLocals(txs) // If locals are disabled, the previous eviction algorithm should apply here too @@ -1112,6 +1195,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) } } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -1140,9 +1224,13 @@ func TestTransactionPendingLimiting(t *testing.T) { if err := pool.addRemoteSync(transaction(i, 100000, key)); err != nil { t.Fatalf("tx %d: failed to add transaction: %v", i, err) } + + pool.pendingMu.RLock() if pool.pending[account].Len() != int(i)+1 { t.Errorf("tx %d: pending pool size mismatch: have %d, want %d", i, pool.pending[account].Len(), i+1) } + pool.pendingMu.RUnlock() + if len(pool.queue) != 0 { t.Errorf("tx %d: queue size mismatch: have %d, want %d", i, pool.queue[account].Len(), 0) } @@ -1195,9 +1283,13 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { pool.AddRemotesSync(txs) pending := 0 + + pool.pendingMu.RLock() for _, list := range pool.pending { pending += list.Len() } + pool.pendingMu.RUnlock() + if pending > int(config.GlobalSlots) { t.Fatalf("total pending transactions overflow allowance: %d > %d", pending, config.GlobalSlots) } @@ -1330,11 +1422,14 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { // Import the batch and verify that limits have been enforced pool.AddRemotesSync(txs) + pool.pendingMu.RLock() for addr, list := range pool.pending { if list.Len() != int(config.AccountSlots) { t.Errorf("addr %x: total pending transactions mismatch: have %d, want %d", addr, list.Len(), config.AccountSlots) } } + pool.pendingMu.RUnlock() + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -1391,15 +1486,19 @@ func TestTransactionPoolRepricing(t *testing.T) { if pending != 7 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7) } + if queued != 3 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) } + if err := validateEvents(events, 7); err != nil { t.Fatalf("original event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // Reprice the pool and check that underpriced transactions get dropped pool.SetGasPrice(big.NewInt(2)) @@ -1407,58 +1506,76 @@ func TestTransactionPoolRepricing(t *testing.T) { if pending != 2 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) } + if queued != 5 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5) } + if err := validateEvents(events, 0); err != nil { t.Fatalf("reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // Check that we can't add the old transactions back - if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced { + + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); err != ErrUnderpriced { + + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } + if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // However we can add local underpriced transactions tx := pricedTransaction(1, 100000, big.NewInt(1), keys[3]) + if err := pool.AddLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } + if pending, _ = pool.Stats(); pending != 3 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) } + if err := validateEvents(events, 1); err != nil { t.Fatalf("post-reprice local event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // And we can fill gaps with properly priced transactions if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } + if err := validateEvents(events, 5); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -1487,6 +1604,7 @@ func TestTransactionPoolRepricingDynamicFee(t *testing.T) { keys[i], _ = crypto.GenerateKey() testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } + // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} @@ -1512,15 +1630,19 @@ func TestTransactionPoolRepricingDynamicFee(t *testing.T) { if pending != 7 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7) } + if queued != 3 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) } + if err := validateEvents(events, 7); err != nil { t.Fatalf("original event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // Reprice the pool and check that underpriced transactions get dropped pool.SetGasPrice(big.NewInt(2)) @@ -1528,64 +1650,87 @@ func TestTransactionPoolRepricingDynamicFee(t *testing.T) { if pending != 2 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) } + if queued != 5 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5) } + if err := validateEvents(events, 0); err != nil { t.Fatalf("reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // Check that we can't add the old transactions back tx := pricedTransaction(1, 100000, big.NewInt(1), keys[0]) - if err := pool.AddRemote(tx); err != ErrUnderpriced { + + if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } + tx = dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.AddRemote(tx); err != ErrUnderpriced { + + if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } + tx = dynamicFeeTx(2, 100000, big.NewInt(1), big.NewInt(1), keys[2]) - if err := pool.AddRemote(tx); err != ErrUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } + if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // However we can add local underpriced transactions tx = dynamicFeeTx(1, 100000, big.NewInt(1), big.NewInt(1), keys[3]) + if err := pool.AddLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } + if pending, _ = pool.Stats(); pending != 3 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) } + if err := validateEvents(events, 1); err != nil { t.Fatalf("post-reprice local event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } + // And we can fill gaps with properly priced transactions tx = pricedTransaction(1, 100000, big.NewInt(2), keys[0]) + if err := pool.AddRemote(tx); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } + tx = dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), keys[1]) + if err := pool.AddRemote(tx); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } + tx = dynamicFeeTx(2, 100000, big.NewInt(2), big.NewInt(2), keys[2]) + if err := pool.AddRemote(tx); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } + if err := validateEvents(events, 5); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -1719,7 +1864,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding an underpriced transaction on block limit fails - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); err != ErrUnderpriced { + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } // Ensure that adding high priced transactions drops cheap ones, but not own @@ -1891,7 +2036,7 @@ func TestTransactionPoolUnderpricingDynamicFee(t *testing.T) { // Ensure that adding an underpriced transaction fails tx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.AddRemote(tx); err != ErrUnderpriced { // Pend K0:0, K0:1, K2:0; Que K1:1 + if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { // Pend K0:0, K0:1, K2:0; Que K1:1 t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) } @@ -1991,7 +2136,7 @@ func TestDualHeapEviction(t *testing.T) { add(false) for baseFee = 0; baseFee <= 1000; baseFee += 100 { - pool.priced.SetBaseFee(big.NewInt(int64(baseFee))) + pool.priced.SetBaseFee(uint256.NewInt(uint64(baseFee))) add(true) check(highCap, "fee cap") add(false) @@ -2020,49 +2165,65 @@ func TestTransactionDeduplication(t *testing.T) { // Create a batch of transactions and add a few of them txs := make([]*types.Transaction, 16) + for i := 0; i < len(txs); i++ { txs[i] = pricedTransaction(uint64(i), 100000, big.NewInt(1), key) } + var firsts []*types.Transaction + for i := 0; i < len(txs); i += 2 { firsts = append(firsts, txs[i]) } + errs := pool.AddRemotesSync(firsts) - if len(errs) != len(firsts) { - t.Fatalf("first add mismatching result count: have %d, want %d", len(errs), len(firsts)) + + if len(errs) != 0 { + t.Fatalf("first add mismatching result count: have %d, want %d", len(errs), 0) } + for i, err := range errs { if err != nil { t.Errorf("add %d failed: %v", i, err) } } + pending, queued := pool.Stats() + if pending != 1 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1) } + if queued != len(txs)/2-1 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, len(txs)/2-1) } + // Try to add all of them now and ensure previous ones error out as knowns errs = pool.AddRemotesSync(txs) - if len(errs) != len(txs) { - t.Fatalf("all add mismatching result count: have %d, want %d", len(errs), len(txs)) + if len(errs) != 0 { + t.Fatalf("all add mismatching result count: have %d, want %d", len(errs), 0) } + for i, err := range errs { if i%2 == 0 && err == nil { t.Errorf("add %d succeeded, should have failed as known", i) } + if i%2 == 1 && err != nil { t.Errorf("add %d failed: %v", i, err) } } + pending, queued = pool.Stats() + if pending != len(txs) { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, len(txs)) } + if queued != 0 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -2096,12 +2257,15 @@ func TestTransactionReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); err != ErrReplaceUnderpriced { + + if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original cheap pending transaction: %v", err) } + if err := validateEvents(events, 2); err != nil { t.Fatalf("cheap replacement event firing failed: %v", err) } @@ -2109,12 +2273,15 @@ func TestTransactionReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced { + + if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } + if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(threshold), key)); err != nil { t.Fatalf("failed to replace original proper pending transaction: %v", err) } + if err := validateEvents(events, 2); err != nil { t.Fatalf("proper replacement event firing failed: %v", err) } @@ -2123,9 +2290,11 @@ func TestTransactionReplacement(t *testing.T) { if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap queued transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); err != ErrReplaceUnderpriced { + + if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original cheap queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original cheap queued transaction: %v", err) } @@ -2133,9 +2302,11 @@ func TestTransactionReplacement(t *testing.T) { if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper queued transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced { + + if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) } + if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(threshold), key)); err != nil { t.Fatalf("failed to replace original proper queued transaction: %v", err) } @@ -2143,6 +2314,7 @@ func TestTransactionReplacement(t *testing.T) { if err := validateEvents(events, 0); err != nil { t.Fatalf("queued replacement event firing failed: %v", err) } + if err := validateTxPoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -2197,7 +2369,7 @@ func TestTransactionReplacementDynamicFee(t *testing.T) { } // 2. Don't bump tip or feecap => discard tx = dynamicFeeTx(nonce, 100001, big.NewInt(2), big.NewInt(1), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original cheap %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) } // 3. Bump both more than min => accept @@ -2220,22 +2392,22 @@ func TestTransactionReplacementDynamicFee(t *testing.T) { } // 6. Bump tip max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold-1), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) } // 7. Bump fee cap max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold-1), big.NewInt(gasTipCap), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) } // 8. Bump tip min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) } // 9. Bump fee cap min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold), big.NewInt(gasTipCap), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { + if err := pool.AddRemote(tx); !errors.Is(err, ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) } // 10. Check events match expected (3 new executable txs during pending, 0 during queue) @@ -2465,6 +2637,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { } // Benchmark the speed of pool validation b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { pool.demoteUnexecutables() } @@ -2496,15 +2669,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { } // Benchmarks the speed of batched transaction insertion. -func BenchmarkPoolBatchInsert100(b *testing.B) { benchmarkPoolBatchInsert(b, 100, false) } -func BenchmarkPoolBatchInsert1000(b *testing.B) { benchmarkPoolBatchInsert(b, 1000, false) } -func BenchmarkPoolBatchInsert10000(b *testing.B) { benchmarkPoolBatchInsert(b, 10000, false) } - -func BenchmarkPoolBatchLocalInsert100(b *testing.B) { benchmarkPoolBatchInsert(b, 100, true) } -func BenchmarkPoolBatchLocalInsert1000(b *testing.B) { benchmarkPoolBatchInsert(b, 1000, true) } -func BenchmarkPoolBatchLocalInsert10000(b *testing.B) { benchmarkPoolBatchInsert(b, 10000, true) } - -func benchmarkPoolBatchInsert(b *testing.B, size int, local bool) { +func BenchmarkPoolBatchInsert(b *testing.B) { // Generate a batch of transactions to enqueue into the pool pool, key := setupTxPool() defer pool.Stop() @@ -2512,21 +2677,153 @@ func benchmarkPoolBatchInsert(b *testing.B, size int, local bool) { account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000)) - batches := make([]types.Transactions, b.N) - for i := 0; i < b.N; i++ { - batches[i] = make(types.Transactions, size) - for j := 0; j < size; j++ { - batches[i][j] = transaction(uint64(size*i+j), 100000, key) - } + const format = "size %d, is local %t" + + cases := []struct { + name string + size int + isLocal bool + }{ + {size: 100, isLocal: false}, + {size: 1000, isLocal: false}, + {size: 10000, isLocal: false}, + + {size: 100, isLocal: true}, + {size: 1000, isLocal: true}, + {size: 10000, isLocal: true}, } + + for i := range cases { + cases[i].name = fmt.Sprintf(format, cases[i].size, cases[i].isLocal) + } + // Benchmark importing the transactions into the queue - b.ResetTimer() - for _, batch := range batches { - if local { - pool.AddLocals(batch) - } else { - pool.AddRemotes(batch) - } + + for _, testCase := range cases { + singleCase := testCase + + b.Run(singleCase.name, func(b *testing.B) { + batches := make([]types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + batches[i] = make(types.Transactions, singleCase.size) + + for j := 0; j < singleCase.size; j++ { + batches[i][j] = transaction(uint64(singleCase.size*i+j), 100000, key) + } + } + + b.ResetTimer() + b.ReportAllocs() + + for _, batch := range batches { + if testCase.isLocal { + pool.AddLocals(batch) + } else { + pool.AddRemotes(batch) + } + } + }) + } +} + +func BenchmarkPoolMining(b *testing.B) { + const format = "size %d" + + cases := []struct { + name string + size int + }{ + {size: 1}, + {size: 5}, + {size: 10}, + {size: 20}, + } + + for i := range cases { + cases[i].name = fmt.Sprintf(format, cases[i].size) + } + + const blockGasLimit = 30_000_000 + + // Benchmark importing the transactions into the queue + + for _, testCase := range cases { + singleCase := testCase + + b.Run(singleCase.name, func(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pendingAddedCh := make(chan struct{}, 1024) + + pool, localKey := setupTxPoolWithConfig(params.TestChainConfig, testTxPoolConfig, txPoolGasLimit, MakeWithPromoteTxCh(pendingAddedCh)) + defer pool.Stop() + + localKeyPub := localKey.PublicKey + account := crypto.PubkeyToAddress(localKeyPub) + + const balanceStr = "1_000_000_000" + balance, ok := big.NewInt(0).SetString(balanceStr, 0) + if !ok { + b.Fatal("incorrect initial balance", balanceStr) + } + + testAddBalance(pool, account, balance) + + signer := types.NewEIP155Signer(big.NewInt(1)) + baseFee := uint256.NewInt(1) + + const batchesSize = 100 + + batches := make([]types.Transactions, batchesSize) + + for i := 0; i < batchesSize; i++ { + batches[i] = make(types.Transactions, singleCase.size) + + for j := 0; j < singleCase.size; j++ { + batches[i][j] = transaction(uint64(singleCase.size*i+j), 100_000, localKey) + } + + for _, batch := range batches { + pool.AddRemotes(batch) + } + } + + var promoted int + + for range pendingAddedCh { + promoted++ + + if promoted >= batchesSize*singleCase.size/2 { + break + } + } + + var total int + + b.ResetTimer() + b.ReportAllocs() + + pendingDurations := make([]time.Duration, b.N) + + var added int + + for i := 0; i < b.N; i++ { + added, pendingDurations[i], _ = mining(b, pool, signer, baseFee, blockGasLimit, i) + total += added + } + + b.StopTimer() + + pendingDurationsFloat := make([]float64, len(pendingDurations)) + + for i, v := range pendingDurations { + pendingDurationsFloat[i] = float64(v.Nanoseconds()) + } + + mean, stddev := stat.MeanStdDev(pendingDurationsFloat, nil) + b.Logf("[%s] pending mean %v, stdev %v, %v-%v", + common.NowMilliseconds(), time.Duration(mean), time.Duration(stddev), time.Duration(floats.Min(pendingDurationsFloat)), time.Duration(floats.Max(pendingDurationsFloat))) + }) } } @@ -2566,26 +2863,319 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { } // Benchmarks the speed of batch transaction insertion in case of multiple accounts. -func BenchmarkPoolMultiAccountBatchInsert(b *testing.B) { +func BenchmarkPoolAccountMultiBatchInsert(b *testing.B) { // Generate a batch of transactions to enqueue into the pool pool, _ := setupTxPool() defer pool.Stop() - b.ReportAllocs() + batches := make(types.Transactions, b.N) + for i := 0; i < b.N; i++ { key, _ := crypto.GenerateKey() account := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(account, big.NewInt(1000000)) + tx := transaction(uint64(0), 100000, key) + batches[i] = tx } + // Benchmark importing the transactions into the queue + b.ReportAllocs() b.ResetTimer() + for _, tx := range batches { pool.AddRemotesSync([]*types.Transaction{tx}) } } +func BenchmarkPoolAccountMultiBatchInsertRace(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pool, _ := setupTxPool() + defer pool.Stop() + + batches := make(types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + tx := transaction(uint64(0), 100000, key) + + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + batches[i] = tx + } + + done := make(chan struct{}) + + go func() { + t := time.NewTicker(time.Microsecond) + defer t.Stop() + + var pending map[common.Address]types.Transactions + + loop: + for { + select { + case <-t.C: + pending = pool.Pending(context.Background(), true) + case <-done: + break loop + } + } + + fmt.Fprint(io.Discard, pending) + }() + + b.ReportAllocs() + b.ResetTimer() + + for _, tx := range batches { + pool.AddRemotesSync([]*types.Transaction{tx}) + } + + close(done) +} + +func BenchmarkPoolAccountMultiBatchInsertNoLockRace(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pendingAddedCh := make(chan struct{}, 1024) + + pool, localKey := setupTxPoolWithConfig(params.TestChainConfig, testTxPoolConfig, txPoolGasLimit, MakeWithPromoteTxCh(pendingAddedCh)) + defer pool.Stop() + + _ = localKey + + batches := make(types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + tx := transaction(uint64(0), 100000, key) + + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + batches[i] = tx + } + + done := make(chan struct{}) + + go func() { + t := time.NewTicker(time.Microsecond) + defer t.Stop() + + var pending map[common.Address]types.Transactions + + for range t.C { + pending = pool.Pending(context.Background(), true) + + if len(pending) >= b.N/2 { + close(done) + + return + } + } + }() + + b.ReportAllocs() + b.ResetTimer() + + for _, tx := range batches { + pool.AddRemotes([]*types.Transaction{tx}) + } + + <-done +} + +func BenchmarkPoolAccountsBatchInsert(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pool, _ := setupTxPool() + defer pool.Stop() + + batches := make(types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + tx := transaction(uint64(0), 100000, key) + + batches[i] = tx + } + + // Benchmark importing the transactions into the queue + b.ReportAllocs() + b.ResetTimer() + + for _, tx := range batches { + _ = pool.AddRemoteSync(tx) + } +} + +func BenchmarkPoolAccountsBatchInsertRace(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pool, _ := setupTxPool() + defer pool.Stop() + + batches := make(types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + tx := transaction(uint64(0), 100000, key) + + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + batches[i] = tx + } + + done := make(chan struct{}) + + go func() { + t := time.NewTicker(time.Microsecond) + defer t.Stop() + + var pending map[common.Address]types.Transactions + + loop: + for { + select { + case <-t.C: + pending = pool.Pending(context.Background(), true) + case <-done: + break loop + } + } + + fmt.Fprint(io.Discard, pending) + }() + + b.ReportAllocs() + b.ResetTimer() + + for _, tx := range batches { + _ = pool.AddRemoteSync(tx) + } + + close(done) +} + +func BenchmarkPoolAccountsBatchInsertNoLockRace(b *testing.B) { + // Generate a batch of transactions to enqueue into the pool + pendingAddedCh := make(chan struct{}, 1024) + + pool, localKey := setupTxPoolWithConfig(params.TestChainConfig, testTxPoolConfig, txPoolGasLimit, MakeWithPromoteTxCh(pendingAddedCh)) + defer pool.Stop() + + _ = localKey + + batches := make(types.Transactions, b.N) + + for i := 0; i < b.N; i++ { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + tx := transaction(uint64(0), 100000, key) + + pool.currentState.AddBalance(account, big.NewInt(1000000)) + + batches[i] = tx + } + + done := make(chan struct{}) + + go func() { + t := time.NewTicker(time.Microsecond) + defer t.Stop() + + var pending map[common.Address]types.Transactions + + for range t.C { + pending = pool.Pending(context.Background(), true) + + if len(pending) >= b.N/2 { + close(done) + + return + } + } + }() + + b.ReportAllocs() + b.ResetTimer() + + for _, tx := range batches { + _ = pool.AddRemote(tx) + } + + <-done +} + +func TestPoolMultiAccountBatchInsertRace(t *testing.T) { + t.Parallel() + + // Generate a batch of transactions to enqueue into the pool + pool, _ := setupTxPool() + defer pool.Stop() + + const n = 5000 + + batches := make(types.Transactions, n) + batchesSecond := make(types.Transactions, n) + + for i := 0; i < n; i++ { + batches[i] = newTxs(pool) + batchesSecond[i] = newTxs(pool) + } + + done := make(chan struct{}) + + go func() { + t := time.NewTicker(time.Microsecond) + defer t.Stop() + + var ( + pending map[common.Address]types.Transactions + total int + ) + + for range t.C { + pending = pool.Pending(context.Background(), true) + total = len(pending) + + _ = pool.Locals() + + if total >= n { + close(done) + + return + } + } + }() + + for _, tx := range batches { + pool.AddRemotesSync([]*types.Transaction{tx}) + } + + for _, tx := range batchesSecond { + pool.AddRemotes([]*types.Transaction{tx}) + } + + <-done +} + +func newTxs(pool *TxPool) *types.Transaction { + key, _ := crypto.GenerateKey() + account := crypto.PubkeyToAddress(key.PublicKey) + tx := transaction(uint64(0), 100000, key) + + pool.currentState.AddBalance(account, big.NewInt(1_000_000_000)) + + return tx +} + type acc struct { nonce uint64 key *ecdsa.PrivateKey @@ -2878,20 +3468,20 @@ func testPoolBatchInsert(t *testing.T, cfg txPoolRapidConfig) { wg.Wait() var ( - addIntoTxPool func(tx []*types.Transaction) []error + addIntoTxPool func(tx *types.Transaction) error totalInBatch int ) for _, tx := range txs.txs { - addIntoTxPool = pool.AddRemotesSync + addIntoTxPool = pool.AddRemoteSync if tx.isLocal { - addIntoTxPool = pool.AddLocals + addIntoTxPool = pool.AddLocal } - err := addIntoTxPool([]*types.Transaction{tx.tx}) - if len(err) != 0 && err[0] != nil { - rt.Log("on adding a transaction to the tx pool", err[0], tx.tx.Gas(), tx.tx.GasPrice(), pool.GasPrice(), getBalance(pool, keys[tx.idx].account)) + err := addIntoTxPool(tx.tx) + if err != nil { + rt.Log("on adding a transaction to the tx pool", err, tx.tx.Gas(), tx.tx.GasPrice(), pool.GasPrice(), getBalance(pool, keys[tx.idx].account)) } } @@ -2930,7 +3520,7 @@ func testPoolBatchInsert(t *testing.T, cfg txPoolRapidConfig) { // check if txPool got stuck if currentTxPoolStats == lastTxPoolStats { - stuckBlocks++ //todo: переписать + stuckBlocks++ //todo: need something better then that } else { stuckBlocks = 0 lastTxPoolStats = currentTxPoolStats @@ -2938,7 +3528,7 @@ func testPoolBatchInsert(t *testing.T, cfg txPoolRapidConfig) { // copy-paste start := time.Now() - pending := pool.Pending(true) + pending := pool.Pending(context.Background(), true) locals := pool.Locals() // from fillTransactions @@ -2956,7 +3546,7 @@ func testPoolBatchInsert(t *testing.T, cfg txPoolRapidConfig) { // check for nonce gaps var lastNonce, currentNonce int - pending = pool.Pending(true) + pending = pool.Pending(context.Background(), true) for txAcc, pendingTxs := range pending { lastNonce = int(pool.Nonce(txAcc)) - len(pendingTxs) - 1 @@ -3026,7 +3616,7 @@ func fillTransactions(ctx context.Context, pool *TxPool, locals []common.Address signer := types.NewLondonSigner(big.NewInt(1)) // fake baseFee - baseFee := big.NewInt(1) + baseFee := uint256.NewInt(1) blockGasLimit := gasLimit @@ -3083,7 +3673,10 @@ func commitTransactions(pool *TxPool, txs *types.TransactionsByPriceAndNonce, bl if tx.Gas() <= blockGasLimit { blockGasLimit -= tx.Gas() + + pool.mu.Lock() pool.removeTx(tx.Hash(), false) + pool.mu.Unlock() txCount++ } else { @@ -3098,3 +3691,885 @@ func MakeWithPromoteTxCh(ch chan struct{}) func(*TxPool) { pool.promoteTxCh = ch } } + +//nolint:thelper +func mining(tb testing.TB, pool *TxPool, signer types.Signer, baseFee *uint256.Int, blockGasLimit uint64, totalBlocks int) (int, time.Duration, time.Duration) { + var ( + localTxsCount int + remoteTxsCount int + localTxs = make(map[common.Address]types.Transactions) + remoteTxs map[common.Address]types.Transactions + total int + ) + + start := time.Now() + + pending := pool.Pending(context.Background(), true) + + pendingDuration := time.Since(start) + + remoteTxs = pending + + locals := pool.Locals() + + pendingLen, queuedLen := pool.Stats() + + for _, account := range locals { + if txs := remoteTxs[account]; len(txs) > 0 { + delete(remoteTxs, account) + + localTxs[account] = txs + } + } + + localTxsCount = len(localTxs) + remoteTxsCount = len(remoteTxs) + + var txLocalCount int + + if localTxsCount > 0 { + txs := types.NewTransactionsByPriceAndNonce(signer, localTxs, baseFee) + + blockGasLimit, txLocalCount = commitTransactions(pool, txs, blockGasLimit) + + total += txLocalCount + } + + var txRemoteCount int + + if remoteTxsCount > 0 { + txs := types.NewTransactionsByPriceAndNonce(signer, remoteTxs, baseFee) + + _, txRemoteCount = commitTransactions(pool, txs, blockGasLimit) + + total += txRemoteCount + } + + miningDuration := time.Since(start) + + tb.Logf("[%s] mining block. block %d. total %d: pending %d(added %d), local %d(added %d), queued %d, localTxsCount %d, remoteTxsCount %d, pending %v, mining %v", + common.NowMilliseconds(), totalBlocks, total, pendingLen, txRemoteCount, localTxsCount, txLocalCount, queuedLen, localTxsCount, remoteTxsCount, pendingDuration, miningDuration) + + return total, pendingDuration, miningDuration +} + +//nolint:paralleltest +func TestPoolMiningDataRaces(t *testing.T) { + if testing.Short() { + t.Skip("only for data race testing") + } + + const format = "size %d, txs ticker %v, api ticker %v" + + cases := []struct { + name string + size int + txsTickerDuration time.Duration + apiTickerDuration time.Duration + }{ + { + size: 1, + txsTickerDuration: 200 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 1, + txsTickerDuration: 400 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 1, + txsTickerDuration: 600 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 1, + txsTickerDuration: 800 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + + { + size: 5, + txsTickerDuration: 200 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 5, + txsTickerDuration: 400 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 5, + txsTickerDuration: 600 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 5, + txsTickerDuration: 800 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + + { + size: 10, + txsTickerDuration: 200 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 10, + txsTickerDuration: 400 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 10, + txsTickerDuration: 600 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 10, + txsTickerDuration: 800 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + + { + size: 20, + txsTickerDuration: 200 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 20, + txsTickerDuration: 400 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 20, + txsTickerDuration: 600 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 20, + txsTickerDuration: 800 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + + { + size: 30, + txsTickerDuration: 200 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 30, + txsTickerDuration: 400 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 30, + txsTickerDuration: 600 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + { + size: 30, + txsTickerDuration: 800 * time.Millisecond, + apiTickerDuration: 10 * time.Millisecond, + }, + } + + for i := range cases { + cases[i].name = fmt.Sprintf(format, cases[i].size, cases[i].txsTickerDuration, cases[i].apiTickerDuration) + } + + //nolint:paralleltest + for _, testCase := range cases { + singleCase := testCase + + t.Run(singleCase.name, func(t *testing.T) { + defer goleak.VerifyNone(t, leak.IgnoreList()...) + + const ( + blocks = 300 + blockGasLimit = 40_000_000 + blockPeriod = time.Second + threads = 10 + batchesSize = 10_000 + timeoutDuration = 10 * blockPeriod + + balanceStr = "1_000_000_000_000" + ) + + apiWithMining(t, balanceStr, batchesSize, singleCase, timeoutDuration, threads, blockPeriod, blocks, blockGasLimit) + }) + } +} + +//nolint:gocognit,thelper +func apiWithMining(tb testing.TB, balanceStr string, batchesSize int, singleCase struct { + name string + size int + txsTickerDuration time.Duration + apiTickerDuration time.Duration +}, timeoutDuration time.Duration, threads int, blockPeriod time.Duration, blocks int, blockGasLimit uint64) { + done := make(chan struct{}) + + var wg sync.WaitGroup + + defer func() { + close(done) + + tb.Logf("[%s] finishing apiWithMining", common.NowMilliseconds()) + + wg.Wait() + + tb.Logf("[%s] apiWithMining finished", common.NowMilliseconds()) + }() + + // Generate a batch of transactions to enqueue into the pool + pendingAddedCh := make(chan struct{}, 1024) + + pool, localKey := setupTxPoolWithConfig(params.TestChainConfig, testTxPoolConfig, txPoolGasLimit, MakeWithPromoteTxCh(pendingAddedCh)) + defer pool.Stop() + + localKeyPub := localKey.PublicKey + account := crypto.PubkeyToAddress(localKeyPub) + + balance, ok := big.NewInt(0).SetString(balanceStr, 0) + if !ok { + tb.Fatal("incorrect initial balance", balanceStr) + } + + testAddBalance(pool, account, balance) + + signer := types.NewEIP155Signer(big.NewInt(1)) + baseFee := uint256.NewInt(1) + + batchesLocal := make([]types.Transactions, batchesSize) + batchesRemote := make([]types.Transactions, batchesSize) + batchesRemotes := make([]types.Transactions, batchesSize) + batchesRemoteSync := make([]types.Transactions, batchesSize) + batchesRemotesSync := make([]types.Transactions, batchesSize) + + for i := 0; i < batchesSize; i++ { + batchesLocal[i] = make(types.Transactions, singleCase.size) + + for j := 0; j < singleCase.size; j++ { + batchesLocal[i][j] = pricedTransaction(uint64(singleCase.size*i+j), 100_000, big.NewInt(int64(i+1)), localKey) + } + + batchesRemote[i] = make(types.Transactions, singleCase.size) + + remoteKey, _ := crypto.GenerateKey() + remoteAddr := crypto.PubkeyToAddress(remoteKey.PublicKey) + testAddBalance(pool, remoteAddr, balance) + + for j := 0; j < singleCase.size; j++ { + batchesRemote[i][j] = pricedTransaction(uint64(j), 100_000, big.NewInt(int64(i+1)), remoteKey) + } + + batchesRemotes[i] = make(types.Transactions, singleCase.size) + + remotesKey, _ := crypto.GenerateKey() + remotesAddr := crypto.PubkeyToAddress(remotesKey.PublicKey) + testAddBalance(pool, remotesAddr, balance) + + for j := 0; j < singleCase.size; j++ { + batchesRemotes[i][j] = pricedTransaction(uint64(j), 100_000, big.NewInt(int64(i+1)), remotesKey) + } + + batchesRemoteSync[i] = make(types.Transactions, singleCase.size) + + remoteSyncKey, _ := crypto.GenerateKey() + remoteSyncAddr := crypto.PubkeyToAddress(remoteSyncKey.PublicKey) + testAddBalance(pool, remoteSyncAddr, balance) + + for j := 0; j < singleCase.size; j++ { + batchesRemoteSync[i][j] = pricedTransaction(uint64(j), 100_000, big.NewInt(int64(i+1)), remoteSyncKey) + } + + batchesRemotesSync[i] = make(types.Transactions, singleCase.size) + + remotesSyncKey, _ := crypto.GenerateKey() + remotesSyncAddr := crypto.PubkeyToAddress(remotesSyncKey.PublicKey) + testAddBalance(pool, remotesSyncAddr, balance) + + for j := 0; j < singleCase.size; j++ { + batchesRemotesSync[i][j] = pricedTransaction(uint64(j), 100_000, big.NewInt(int64(i+1)), remotesSyncKey) + } + } + + tb.Logf("[%s] starting goroutines", common.NowMilliseconds()) + + txsTickerDuration := singleCase.txsTickerDuration + apiTickerDuration := singleCase.apiTickerDuration + + // locals + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping AddLocal(s)", common.NowMilliseconds()) + + wg.Done() + + tb.Logf("[%s] stopped AddLocal(s)", common.NowMilliseconds()) + }() + + tb.Logf("[%s] starting AddLocal(s)", common.NowMilliseconds()) + + for _, batch := range batchesLocal { + batch := batch + + select { + case <-done: + return + default: + } + + if rand.Int()%2 == 0 { + runWithTimeout(tb, func(_ chan struct{}) { + errs := pool.AddLocals(batch) + if len(errs) != 0 { + tb.Logf("[%s] AddLocals error, %v", common.NowMilliseconds(), errs) + } + }, done, "AddLocals", timeoutDuration, 0, 0) + } else { + for _, tx := range batch { + tx := tx + + runWithTimeout(tb, func(_ chan struct{}) { + err := pool.AddLocal(tx) + if err != nil { + tb.Logf("[%s] AddLocal error %s", common.NowMilliseconds(), err) + } + }, done, "AddLocal", timeoutDuration, 0, 0) + + time.Sleep(txsTickerDuration) + } + } + + time.Sleep(txsTickerDuration) + } + }() + + // remotes + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping AddRemotes", common.NowMilliseconds()) + + wg.Done() + + tb.Logf("[%s] stopped AddRemotes", common.NowMilliseconds()) + }() + + addTransactionsBatches(tb, batchesRemotes, getFnForBatches(pool.AddRemotes), done, timeoutDuration, txsTickerDuration, "AddRemotes", 0) + }() + + // remote + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping AddRemote", common.NowMilliseconds()) + + wg.Done() + + tb.Logf("[%s] stopped AddRemote", common.NowMilliseconds()) + }() + + addTransactions(tb, batchesRemote, pool.AddRemote, done, timeoutDuration, txsTickerDuration, "AddRemote", 0) + }() + + // sync + // remotes + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping AddRemotesSync", common.NowMilliseconds()) + + wg.Done() + + tb.Logf("[%s] stopped AddRemotesSync", common.NowMilliseconds()) + }() + + addTransactionsBatches(tb, batchesRemotesSync, getFnForBatches(pool.AddRemotesSync), done, timeoutDuration, txsTickerDuration, "AddRemotesSync", 0) + }() + + // remote + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping AddRemoteSync", common.NowMilliseconds()) + + wg.Done() + + tb.Logf("[%s] stopped AddRemoteSync", common.NowMilliseconds()) + }() + + addTransactions(tb, batchesRemoteSync, pool.AddRemoteSync, done, timeoutDuration, txsTickerDuration, "AddRemoteSync", 0) + }() + + // tx pool API + for i := 0; i < threads; i++ { + i := i + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Pending-no-tips, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Pending-no-tips, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + p := pool.Pending(context.Background(), false) + fmt.Fprint(io.Discard, p) + }, done, "Pending-no-tips", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Pending-with-tips, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Pending-with-tips, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + p := pool.Pending(context.Background(), true) + fmt.Fprint(io.Discard, p) + }, done, "Pending-with-tips", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Locals, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Locals, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + l := pool.Locals() + fmt.Fprint(io.Discard, l) + }, done, "Locals", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Content, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Content, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + p, q := pool.Content() + fmt.Fprint(io.Discard, p, q) + }, done, "Content", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping GasPriceUint256, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped GasPriceUint256, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + res := pool.GasPriceUint256() + fmt.Fprint(io.Discard, res) + }, done, "GasPriceUint256", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping GasPrice, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped GasPrice, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + res := pool.GasPrice() + fmt.Fprint(io.Discard, res) + }, done, "GasPrice", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping SetGasPrice, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped SetGasPrice, , thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + pool.SetGasPrice(pool.GasPrice()) + }, done, "SetGasPrice", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping ContentFrom, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped ContentFrom, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + p, q := pool.ContentFrom(account) + fmt.Fprint(io.Discard, p, q) + }, done, "ContentFrom", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Has, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Has, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + res := pool.Has(batchesRemotes[0][0].Hash()) + fmt.Fprint(io.Discard, res) + }, done, "Has", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Get, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Get, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + tx := pool.Get(batchesRemotes[0][0].Hash()) + fmt.Fprint(io.Discard, tx == nil) + }, done, "Get", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Nonce, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Nonce, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + res := pool.Nonce(account) + fmt.Fprint(io.Discard, res) + }, done, "Nonce", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Stats, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Stats, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + p, q := pool.Stats() + fmt.Fprint(io.Discard, p, q) + }, done, "Stats", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping Status, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped Status, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(_ chan struct{}) { + st := pool.Status([]common.Hash{batchesRemotes[1][0].Hash()}) + fmt.Fprint(io.Discard, st) + }, done, "Status", apiTickerDuration, timeoutDuration, i) + }() + + wg.Add(1) + + go func() { + defer func() { + tb.Logf("[%s] stopping SubscribeNewTxsEvent, thread %d", common.NowMilliseconds(), i) + + wg.Done() + + tb.Logf("[%s] stopped SubscribeNewTxsEvent, thread %d", common.NowMilliseconds(), i) + }() + + runWithTicker(tb, func(c chan struct{}) { + ch := make(chan NewTxsEvent, 10) + sub := pool.SubscribeNewTxsEvent(ch) + + if sub == nil { + return + } + + defer sub.Unsubscribe() + + select { + case <-done: + return + case <-c: + case res := <-ch: + fmt.Fprint(io.Discard, res) + } + + }, done, "SubscribeNewTxsEvent", apiTickerDuration, timeoutDuration, i) + }() + } + + // wait for the start + tb.Logf("[%s] before the first propagated transaction", common.NowMilliseconds()) + <-pendingAddedCh + tb.Logf("[%s] after the first propagated transaction", common.NowMilliseconds()) + + var ( + totalTxs int + totalBlocks int + ) + + pendingDurations := make([]time.Duration, 0, blocks) + + var ( + added int + pendingDuration time.Duration + miningDuration time.Duration + diff time.Duration + ) + + for { + added, pendingDuration, miningDuration = mining(tb, pool, signer, baseFee, blockGasLimit, totalBlocks) + + totalTxs += added + + pendingDurations = append(pendingDurations, pendingDuration) + + totalBlocks++ + + if totalBlocks > blocks { + fmt.Fprint(io.Discard, totalTxs) + break + } + + diff = blockPeriod - miningDuration + if diff > 0 { + time.Sleep(diff) + } + } + + pendingDurationsFloat := make([]float64, len(pendingDurations)) + + for i, v := range pendingDurations { + pendingDurationsFloat[i] = float64(v.Nanoseconds()) + } + + mean, stddev := stat.MeanStdDev(pendingDurationsFloat, nil) + tb.Logf("[%s] pending mean %v, stddev %v, %v-%v", + common.NowMilliseconds(), time.Duration(mean), time.Duration(stddev), time.Duration(floats.Min(pendingDurationsFloat)), time.Duration(floats.Max(pendingDurationsFloat))) +} + +func addTransactionsBatches(tb testing.TB, batches []types.Transactions, fn func(types.Transactions) error, done chan struct{}, timeoutDuration time.Duration, tickerDuration time.Duration, name string, thread int) { + tb.Helper() + + tb.Logf("[%s] starting %s", common.NowMilliseconds(), name) + + defer func() { + tb.Logf("[%s] stop %s", common.NowMilliseconds(), name) + }() + + for _, batch := range batches { + batch := batch + + select { + case <-done: + return + default: + } + + runWithTimeout(tb, func(_ chan struct{}) { + err := fn(batch) + if err != nil { + tb.Logf("[%s] %s error: %s", common.NowMilliseconds(), name, err) + } + }, done, name, timeoutDuration, 0, thread) + + time.Sleep(tickerDuration) + } +} + +func addTransactions(tb testing.TB, batches []types.Transactions, fn func(*types.Transaction) error, done chan struct{}, timeoutDuration time.Duration, tickerDuration time.Duration, name string, thread int) { + tb.Helper() + + tb.Logf("[%s] starting %s", common.NowMilliseconds(), name) + + defer func() { + tb.Logf("[%s] stop %s", common.NowMilliseconds(), name) + }() + + for _, batch := range batches { + for _, tx := range batch { + tx := tx + + select { + case <-done: + return + default: + } + + runWithTimeout(tb, func(_ chan struct{}) { + err := fn(tx) + if err != nil { + tb.Logf("%s error: %s", name, err) + } + }, done, name, timeoutDuration, 0, thread) + + time.Sleep(tickerDuration) + } + + time.Sleep(tickerDuration) + } +} + +func getFnForBatches(fn func([]*types.Transaction) []error) func(types.Transactions) error { + return func(batch types.Transactions) error { + errs := fn(batch) + if len(errs) != 0 { + return errs[0] + } + + return nil + } +} + +//nolint:unparam +func runWithTicker(tb testing.TB, fn func(c chan struct{}), done chan struct{}, name string, tickerDuration, timeoutDuration time.Duration, thread int) { + tb.Helper() + + select { + case <-done: + tb.Logf("[%s] Short path. finishing outer runWithTicker for %q, thread %d", common.NowMilliseconds(), name, thread) + + return + default: + } + + defer func() { + tb.Logf("[%s] finishing outer runWithTicker for %q, thread %d", common.NowMilliseconds(), name, thread) + }() + + localTicker := time.NewTicker(tickerDuration) + defer localTicker.Stop() + + n := 0 + + for range localTicker.C { + select { + case <-done: + return + default: + } + + runWithTimeout(tb, fn, done, name, timeoutDuration, n, thread) + + n++ + } +} + +func runWithTimeout(tb testing.TB, fn func(chan struct{}), outerDone chan struct{}, name string, timeoutDuration time.Duration, n, thread int) { + tb.Helper() + + select { + case <-outerDone: + tb.Logf("[%s] Short path. exiting inner runWithTimeout by outer exit event for %q, thread %d, iteration %d", common.NowMilliseconds(), name, thread, n) + + return + default: + } + + timeout := time.NewTimer(timeoutDuration) + defer timeout.Stop() + + doneCh := make(chan struct{}) + + isError := new(int32) + *isError = 0 + + go func() { + defer close(doneCh) + + select { + case <-outerDone: + return + default: + fn(doneCh) + } + }() + + const isDebug = false + + var stack string + + select { + case <-outerDone: + tb.Logf("[%s] exiting inner runWithTimeout by outer exit event for %q, thread %d, iteration %d", common.NowMilliseconds(), name, thread, n) + case <-doneCh: + // only for debug + //tb.Logf("[%s] exiting inner runWithTimeout by successful call for %q, thread %d, iteration %d", common.NowMilliseconds(), name, thread, n) + case <-timeout.C: + atomic.StoreInt32(isError, 1) + + if isDebug { + stack = string(debug.Stack(true)) + } + + tb.Errorf("[%s] %s timeouted, thread %d, iteration %d. Stack %s", common.NowMilliseconds(), name, thread, n, stack) + } +} diff --git a/core/types/access_list_tx.go b/core/types/access_list_tx.go index 8ad5e739e9..509f86b622 100644 --- a/core/types/access_list_tx.go +++ b/core/types/access_list_tx.go @@ -19,6 +19,8 @@ package types import ( "math/big" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" ) @@ -44,15 +46,16 @@ func (al AccessList) StorageKeys() int { // AccessListTx is the data of EIP-2930 access list transactions. type AccessListTx struct { - ChainID *big.Int // destination chain ID - Nonce uint64 // nonce of sender account - GasPrice *big.Int // wei per gas - Gas uint64 // gas limit - To *common.Address `rlp:"nil"` // nil means contract creation - Value *big.Int // wei amount - Data []byte // contract invocation input data - AccessList AccessList // EIP-2930 access list - V, R, S *big.Int // signature values + ChainID *big.Int // destination chain ID + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + gasPriceUint256 *uint256.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + AccessList AccessList // EIP-2930 access list + V, R, S *big.Int // signature values } // copy creates a deep copy of the transaction data and initializes all fields. @@ -80,6 +83,12 @@ func (tx *AccessListTx) copy() TxData { } if tx.GasPrice != nil { cpy.GasPrice.Set(tx.GasPrice) + + if cpy.gasPriceUint256 != nil { + cpy.gasPriceUint256.Set(tx.gasPriceUint256) + } else { + cpy.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + } } if tx.V != nil { cpy.V.Set(tx.V) @@ -100,11 +109,39 @@ func (tx *AccessListTx) accessList() AccessList { return tx.AccessList } func (tx *AccessListTx) data() []byte { return tx.Data } func (tx *AccessListTx) gas() uint64 { return tx.Gas } func (tx *AccessListTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *AccessListTx) value() *big.Int { return tx.Value } -func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } -func (tx *AccessListTx) to() *common.Address { return tx.To } +func (tx *AccessListTx) gasPriceU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} + +func (tx *AccessListTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasTipCapU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} +func (tx *AccessListTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *AccessListTx) gasFeeCapU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} +func (tx *AccessListTx) value() *big.Int { return tx.Value } +func (tx *AccessListTx) nonce() uint64 { return tx.Nonce } +func (tx *AccessListTx) to() *common.Address { return tx.To } func (tx *AccessListTx) rawSignatureValues() (v, r, s *big.Int) { return tx.V, tx.R, tx.S diff --git a/core/types/dynamic_fee_tx.go b/core/types/dynamic_fee_tx.go index 53f246ea1f..532544d54e 100644 --- a/core/types/dynamic_fee_tx.go +++ b/core/types/dynamic_fee_tx.go @@ -19,19 +19,23 @@ package types import ( "math/big" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" ) type DynamicFeeTx struct { - ChainID *big.Int - Nonce uint64 - GasTipCap *big.Int // a.k.a. maxPriorityFeePerGas - GasFeeCap *big.Int // a.k.a. maxFeePerGas - Gas uint64 - To *common.Address `rlp:"nil"` // nil means contract creation - Value *big.Int - Data []byte - AccessList AccessList + ChainID *big.Int + Nonce uint64 + GasTipCap *big.Int // a.k.a. maxPriorityFeePerGas + gasTipCapUint256 *uint256.Int // a.k.a. maxPriorityFeePerGas + GasFeeCap *big.Int // a.k.a. maxFeePerGas + gasFeeCapUint256 *uint256.Int // a.k.a. maxFeePerGas + Gas uint64 + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int + Data []byte + AccessList AccessList // Signature values V *big.Int `json:"v" gencodec:"required"` @@ -65,9 +69,21 @@ func (tx *DynamicFeeTx) copy() TxData { } if tx.GasTipCap != nil { cpy.GasTipCap.Set(tx.GasTipCap) + + if cpy.gasTipCapUint256 != nil { + cpy.gasTipCapUint256.Set(tx.gasTipCapUint256) + } else { + cpy.gasTipCapUint256, _ = uint256.FromBig(tx.GasTipCap) + } } if tx.GasFeeCap != nil { cpy.GasFeeCap.Set(tx.GasFeeCap) + + if cpy.gasFeeCapUint256 != nil { + cpy.gasFeeCapUint256.Set(tx.gasFeeCapUint256) + } else { + cpy.gasFeeCapUint256, _ = uint256.FromBig(tx.GasFeeCap) + } } if tx.V != nil { cpy.V.Set(tx.V) @@ -88,11 +104,38 @@ func (tx *DynamicFeeTx) accessList() AccessList { return tx.AccessList } func (tx *DynamicFeeTx) data() []byte { return tx.Data } func (tx *DynamicFeeTx) gas() uint64 { return tx.Gas } func (tx *DynamicFeeTx) gasFeeCap() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } -func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } -func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } -func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } -func (tx *DynamicFeeTx) to() *common.Address { return tx.To } +func (tx *DynamicFeeTx) gasFeeCapU256() *uint256.Int { + if tx.gasFeeCapUint256 != nil { + return tx.gasFeeCapUint256 + } + + tx.gasFeeCapUint256, _ = uint256.FromBig(tx.GasFeeCap) + + return tx.gasFeeCapUint256 +} +func (tx *DynamicFeeTx) gasTipCap() *big.Int { return tx.GasTipCap } +func (tx *DynamicFeeTx) gasTipCapU256() *uint256.Int { + if tx.gasTipCapUint256 != nil { + return tx.gasTipCapUint256 + } + + tx.gasTipCapUint256, _ = uint256.FromBig(tx.GasTipCap) + + return tx.gasTipCapUint256 +} +func (tx *DynamicFeeTx) gasPrice() *big.Int { return tx.GasFeeCap } +func (tx *DynamicFeeTx) gasPriceU256() *uint256.Int { + if tx.gasFeeCapUint256 != nil { + return tx.gasTipCapUint256 + } + + tx.gasFeeCapUint256, _ = uint256.FromBig(tx.GasFeeCap) + + return tx.gasFeeCapUint256 +} +func (tx *DynamicFeeTx) value() *big.Int { return tx.Value } +func (tx *DynamicFeeTx) nonce() uint64 { return tx.Nonce } +func (tx *DynamicFeeTx) to() *common.Address { return tx.To } func (tx *DynamicFeeTx) rawSignatureValues() (v, r, s *big.Int) { return tx.V, tx.R, tx.S diff --git a/core/types/legacy_tx.go b/core/types/legacy_tx.go index cb86bed772..72fcd34fa5 100644 --- a/core/types/legacy_tx.go +++ b/core/types/legacy_tx.go @@ -19,18 +19,21 @@ package types import ( "math/big" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" ) // LegacyTx is the transaction data of regular Ethereum transactions. type LegacyTx struct { - Nonce uint64 // nonce of sender account - GasPrice *big.Int // wei per gas - Gas uint64 // gas limit - To *common.Address `rlp:"nil"` // nil means contract creation - Value *big.Int // wei amount - Data []byte // contract invocation input data - V, R, S *big.Int // signature values + Nonce uint64 // nonce of sender account + GasPrice *big.Int // wei per gas + gasPriceUint256 *uint256.Int // wei per gas + Gas uint64 // gas limit + To *common.Address `rlp:"nil"` // nil means contract creation + Value *big.Int // wei amount + Data []byte // contract invocation input data + V, R, S *big.Int // signature values } // NewTransaction creates an unsigned legacy transaction. @@ -77,6 +80,12 @@ func (tx *LegacyTx) copy() TxData { } if tx.GasPrice != nil { cpy.GasPrice.Set(tx.GasPrice) + + if cpy.gasPriceUint256 != nil { + cpy.gasPriceUint256.Set(tx.gasPriceUint256) + } else { + cpy.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + } } if tx.V != nil { cpy.V.Set(tx.V) @@ -97,11 +106,38 @@ func (tx *LegacyTx) accessList() AccessList { return nil } func (tx *LegacyTx) data() []byte { return tx.Data } func (tx *LegacyTx) gas() uint64 { return tx.Gas } func (tx *LegacyTx) gasPrice() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } -func (tx *LegacyTx) value() *big.Int { return tx.Value } -func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } -func (tx *LegacyTx) to() *common.Address { return tx.To } +func (tx *LegacyTx) gasPriceU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} +func (tx *LegacyTx) gasTipCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasTipCapU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} +func (tx *LegacyTx) gasFeeCap() *big.Int { return tx.GasPrice } +func (tx *LegacyTx) gasFeeCapU256() *uint256.Int { + if tx.gasPriceUint256 != nil { + return tx.gasPriceUint256 + } + + tx.gasPriceUint256, _ = uint256.FromBig(tx.GasPrice) + + return tx.gasPriceUint256 +} +func (tx *LegacyTx) value() *big.Int { return tx.Value } +func (tx *LegacyTx) nonce() uint64 { return tx.Nonce } +func (tx *LegacyTx) to() *common.Address { return tx.To } func (tx *LegacyTx) rawSignatureValues() (v, r, s *big.Int) { return tx.V, tx.R, tx.S diff --git a/core/types/transaction.go b/core/types/transaction.go index e0e52f25bc..9b89f12517 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -25,6 +25,8 @@ import ( "sync/atomic" "time" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" @@ -53,9 +55,9 @@ type Transaction struct { time time.Time // Time first seen locally (spam avoidance) // caches - hash atomic.Value - size atomic.Value - from atomic.Value + hash atomic.Pointer[common.Hash] + size atomic.Pointer[common.StorageSize] + from atomic.Pointer[sigCache] } // NewTx creates a new transaction. @@ -77,8 +79,11 @@ type TxData interface { data() []byte gas() uint64 gasPrice() *big.Int + gasPriceU256() *uint256.Int gasTipCap() *big.Int + gasTipCapU256() *uint256.Int gasFeeCap() *big.Int + gasFeeCapU256() *uint256.Int value() *big.Int nonce() uint64 to() *common.Address @@ -194,7 +199,8 @@ func (tx *Transaction) setDecoded(inner TxData, size int) { tx.inner = inner tx.time = time.Now() if size > 0 { - tx.size.Store(common.StorageSize(size)) + v := float64(size) + tx.size.Store((*common.StorageSize)(&v)) } } @@ -265,16 +271,23 @@ func (tx *Transaction) AccessList() AccessList { return tx.inner.accessList() } func (tx *Transaction) Gas() uint64 { return tx.inner.gas() } // GasPrice returns the gas price of the transaction. -func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.inner.gasPrice()) } +func (tx *Transaction) GasPrice() *big.Int { return new(big.Int).Set(tx.inner.gasPrice()) } +func (tx *Transaction) GasPriceRef() *big.Int { return tx.inner.gasPrice() } +func (tx *Transaction) GasPriceUint() *uint256.Int { return tx.inner.gasPriceU256() } // GasTipCap returns the gasTipCap per gas of the transaction. -func (tx *Transaction) GasTipCap() *big.Int { return new(big.Int).Set(tx.inner.gasTipCap()) } +func (tx *Transaction) GasTipCap() *big.Int { return new(big.Int).Set(tx.inner.gasTipCap()) } +func (tx *Transaction) GasTipCapRef() *big.Int { return tx.inner.gasTipCap() } +func (tx *Transaction) GasTipCapUint() *uint256.Int { return tx.inner.gasTipCapU256() } // GasFeeCap returns the fee cap per gas of the transaction. -func (tx *Transaction) GasFeeCap() *big.Int { return new(big.Int).Set(tx.inner.gasFeeCap()) } +func (tx *Transaction) GasFeeCap() *big.Int { return new(big.Int).Set(tx.inner.gasFeeCap()) } +func (tx *Transaction) GasFeeCapRef() *big.Int { return tx.inner.gasFeeCap() } +func (tx *Transaction) GasFeeCapUint() *uint256.Int { return tx.inner.gasFeeCapU256() } // Value returns the ether amount of the transaction. -func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } +func (tx *Transaction) Value() *big.Int { return new(big.Int).Set(tx.inner.value()) } +func (tx *Transaction) ValueRef() *big.Int { return tx.inner.value() } // Nonce returns the sender account nonce of the transaction. func (tx *Transaction) Nonce() uint64 { return tx.inner.nonce() } @@ -287,9 +300,19 @@ func (tx *Transaction) To() *common.Address { // Cost returns gas * gasPrice + value. func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) - total.Add(total, tx.Value()) - return total + gasPrice, _ := uint256.FromBig(tx.GasPriceRef()) + gasPrice.Mul(gasPrice, uint256.NewInt(tx.Gas())) + value, _ := uint256.FromBig(tx.ValueRef()) + + return gasPrice.Add(gasPrice, value).ToBig() +} + +func (tx *Transaction) CostUint() *uint256.Int { + gasPrice, _ := uint256.FromBig(tx.GasPriceRef()) + gasPrice.Mul(gasPrice, uint256.NewInt(tx.Gas())) + value, _ := uint256.FromBig(tx.ValueRef()) + + return gasPrice.Add(gasPrice, value) } // RawSignatureValues returns the V, R, S signature values of the transaction. @@ -303,11 +326,18 @@ func (tx *Transaction) GasFeeCapCmp(other *Transaction) int { return tx.inner.gasFeeCap().Cmp(other.inner.gasFeeCap()) } -// GasFeeCapIntCmp compares the fee cap of the transaction against the given fee cap. func (tx *Transaction) GasFeeCapIntCmp(other *big.Int) int { return tx.inner.gasFeeCap().Cmp(other) } +func (tx *Transaction) GasFeeCapUIntCmp(other *uint256.Int) int { + return tx.inner.gasFeeCapU256().Cmp(other) +} + +func (tx *Transaction) GasFeeCapUIntLt(other *uint256.Int) bool { + return tx.inner.gasFeeCapU256().Lt(other) +} + // GasTipCapCmp compares the gasTipCap of two transactions. func (tx *Transaction) GasTipCapCmp(other *Transaction) int { return tx.inner.gasTipCap().Cmp(other.inner.gasTipCap()) @@ -318,6 +348,14 @@ func (tx *Transaction) GasTipCapIntCmp(other *big.Int) int { return tx.inner.gasTipCap().Cmp(other) } +func (tx *Transaction) GasTipCapUIntCmp(other *uint256.Int) int { + return tx.inner.gasTipCapU256().Cmp(other) +} + +func (tx *Transaction) GasTipCapUIntLt(other *uint256.Int) bool { + return tx.inner.gasTipCapU256().Lt(other) +} + // EffectiveGasTip returns the effective miner gasTipCap for the given base fee. // Note: if the effective gasTipCap is negative, this method returns both error // the actual negative value, _and_ ErrGasFeeCapTooLow @@ -356,10 +394,73 @@ func (tx *Transaction) EffectiveGasTipIntCmp(other *big.Int, baseFee *big.Int) i return tx.EffectiveGasTipValue(baseFee).Cmp(other) } +func (tx *Transaction) EffectiveGasTipUintCmp(other *uint256.Int, baseFee *uint256.Int) int { + if baseFee == nil { + return tx.GasTipCapUIntCmp(other) + } + + return tx.EffectiveGasTipValueUint(baseFee).Cmp(other) +} + +func (tx *Transaction) EffectiveGasTipUintLt(other *uint256.Int, baseFee *uint256.Int) bool { + if baseFee == nil { + return tx.GasTipCapUIntLt(other) + } + + return tx.EffectiveGasTipValueUint(baseFee).Lt(other) +} + +func (tx *Transaction) EffectiveGasTipTxUintCmp(other *Transaction, baseFee *uint256.Int) int { + if baseFee == nil { + return tx.inner.gasTipCapU256().Cmp(other.inner.gasTipCapU256()) + } + + return tx.EffectiveGasTipValueUint(baseFee).Cmp(other.EffectiveGasTipValueUint(baseFee)) +} + +func (tx *Transaction) EffectiveGasTipValueUint(baseFee *uint256.Int) *uint256.Int { + effectiveTip, _ := tx.EffectiveGasTipUnit(baseFee) + return effectiveTip +} + +func (tx *Transaction) EffectiveGasTipUnit(baseFee *uint256.Int) (*uint256.Int, error) { + if baseFee == nil { + return tx.GasFeeCapUint(), nil + } + + var err error + + gasFeeCap := tx.GasFeeCapUint().Clone() + + if gasFeeCap.Lt(baseFee) { + err = ErrGasFeeCapTooLow + } + + gasTipCapUint := tx.GasTipCapUint() + + if gasFeeCap.Lt(gasTipCapUint) { + return gasFeeCap, err + } + + if gasFeeCap.Lt(gasTipCapUint) && baseFee.IsZero() { + return gasFeeCap, err + } + + gasFeeCap.Sub(gasFeeCap, baseFee) + + if gasFeeCap.Gt(gasTipCapUint) || gasFeeCap.Eq(gasTipCapUint) { + gasFeeCap.Add(gasFeeCap, baseFee) + + return gasTipCapUint, err + } + + return gasFeeCap, err +} + // Hash returns the transaction hash. func (tx *Transaction) Hash() common.Hash { if hash := tx.hash.Load(); hash != nil { - return hash.(common.Hash) + return *hash } var h common.Hash @@ -368,7 +469,9 @@ func (tx *Transaction) Hash() common.Hash { } else { h = prefixedRlpHash(tx.Type(), tx.inner) } - tx.hash.Store(h) + + tx.hash.Store(&h) + return h } @@ -376,11 +479,14 @@ func (tx *Transaction) Hash() common.Hash { // encoding and returning it, or returning a previously cached value. func (tx *Transaction) Size() common.StorageSize { if size := tx.size.Load(); size != nil { - return size.(common.StorageSize) + return *size } + c := writeCounter(0) + rlp.Encode(&c, &tx.inner) - tx.size.Store(common.StorageSize(c)) + tx.size.Store((*common.StorageSize)(&c)) + return common.StorageSize(c) } @@ -444,14 +550,14 @@ func (s TxByNonce) Swap(i, j int) { s[i], s[j] = s[j], s[i] } // TxWithMinerFee wraps a transaction with its gas price or effective miner gasTipCap type TxWithMinerFee struct { tx *Transaction - minerFee *big.Int + minerFee *uint256.Int } // NewTxWithMinerFee creates a wrapped transaction, calculating the effective // miner gasTipCap if a base fee is provided. // Returns error in case of a negative effective miner gasTipCap. -func NewTxWithMinerFee(tx *Transaction, baseFee *big.Int) (*TxWithMinerFee, error) { - minerFee, err := tx.EffectiveGasTip(baseFee) +func NewTxWithMinerFee(tx *Transaction, baseFee *uint256.Int) (*TxWithMinerFee, error) { + minerFee, err := tx.EffectiveGasTipUnit(baseFee) if err != nil { return nil, err } @@ -496,7 +602,7 @@ type TransactionsByPriceAndNonce struct { txs map[common.Address]Transactions // Per account nonce-sorted list of transactions heads TxByPriceAndTime // Next transaction for each unique account (price heap) signer Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + baseFee *uint256.Int // Current base fee } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -504,6 +610,7 @@ type TransactionsByPriceAndNonce struct { // // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. +/* func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, baseFee *big.Int) *TransactionsByPriceAndNonce { // Initialize a price and received time based heap with the head transactions heads := make(TxByPriceAndTime, 0, len(txs)) @@ -524,6 +631,39 @@ func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transa } heap.Init(&heads) + // Assemble and return the transaction set + return &TransactionsByPriceAndNonce{ + txs: txs, + heads: heads, + signer: signer, + baseFee: baseFee, + } +}*/ + +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, baseFee *uint256.Int) *TransactionsByPriceAndNonce { + // Initialize a price and received time based heap with the head transactions + heads := make(TxByPriceAndTime, 0, len(txs)) + + for from, accTxs := range txs { + if len(accTxs) == 0 { + continue + } + + acc, _ := Sender(signer, accTxs[0]) + wrapped, err := NewTxWithMinerFee(accTxs[0], baseFee) + + // Remove transaction if sender doesn't match from, or if wrapping fails. + if acc != from || err != nil { + delete(txs, from) + continue + } + + heads = append(heads, wrapped) + txs[from] = accTxs[1:] + } + + heap.Init(&heads) + // Assemble and return the transaction set return &TransactionsByPriceAndNonce{ txs: txs, diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 1d0d2a4c75..959aba637a 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -130,12 +130,11 @@ func MustSignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) *Transaction // not match the signer used in the current call. func Sender(signer Signer, tx *Transaction) (common.Address, error) { if sc := tx.from.Load(); sc != nil { - sigCache := sc.(sigCache) // If the signer used to derive from in a previous // call is not the same as used current, invalidate // the cache. - if sigCache.signer.Equal(signer) { - return sigCache.from, nil + if sc.signer.Equal(signer) { + return sc.from, nil } } @@ -143,7 +142,9 @@ func Sender(signer Signer, tx *Transaction) (common.Address, error) { if err != nil { return common.Address{}, err } - tx.from.Store(sigCache{signer: signer, from: addr}) + + tx.from.Store(&sigCache{signer: signer, from: addr}) + return addr, nil } @@ -461,10 +462,10 @@ func (fs FrontierSigner) SignatureValues(tx *Transaction, sig []byte) (r, s, v * func (fs FrontierSigner) Hash(tx *Transaction) common.Hash { return rlpHash([]interface{}{ tx.Nonce(), - tx.GasPrice(), + tx.GasPriceRef(), tx.Gas(), tx.To(), - tx.Value(), + tx.ValueRef(), tx.Data(), }) } diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index a4755675cd..255a7b76b4 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -27,7 +27,10 @@ import ( "testing" "time" + "github.com/holiman/uint256" + "github.com/ethereum/go-ethereum/common" + cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -272,14 +275,22 @@ func TestTransactionPriceNonceSort1559(t *testing.T) { // Tests that transactions can be correctly sorted according to their price in // decreasing order, but at the same time with increasing nonces when issued by // the same account. -func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { +// +//nolint:gocognit,thelper +func testTransactionPriceNonceSort(t *testing.T, baseFeeBig *big.Int) { // Generate a batch of accounts to start with keys := make([]*ecdsa.PrivateKey, 25) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() } + signer := LatestSignerForChainID(common.Big1) + var baseFee *uint256.Int + if baseFeeBig != nil { + baseFee = cmath.FromBig(baseFeeBig) + } + // Generate a batch of transactions with overlapping values, but shifted nonces groups := map[common.Address]Transactions{} expectedCount := 0 @@ -308,7 +319,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { GasTipCap: big.NewInt(int64(rand.Intn(gasFeeCap + 1))), Data: nil, }) - if count == 25 && int64(gasFeeCap) < baseFee.Int64() { + if count == 25 && uint64(gasFeeCap) < baseFee.Uint64() { count = i } } @@ -341,12 +352,25 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { t.Errorf("invalid nonce ordering: tx #%d (A=%x N=%v) < tx #%d (A=%x N=%v)", i, fromi[:4], txi.Nonce(), i+j, fromj[:4], txj.Nonce()) } } + // If the next tx has different from account, the price must be lower than the current one if i+1 < len(txs) { next := txs[i+1] fromNext, _ := Sender(signer, next) - tip, err := txi.EffectiveGasTip(baseFee) - nextTip, nextErr := next.EffectiveGasTip(baseFee) + tip, err := txi.EffectiveGasTipUnit(baseFee) + nextTip, nextErr := next.EffectiveGasTipUnit(baseFee) + + tipBig, _ := txi.EffectiveGasTip(baseFeeBig) + nextTipBig, _ := next.EffectiveGasTip(baseFeeBig) + + if tip.Cmp(cmath.FromBig(tipBig)) != 0 { + t.Fatalf("EffectiveGasTip incorrect. uint256 %q, big.Int %q, baseFee %q, baseFeeBig %q", tip.String(), tipBig.String(), baseFee.String(), baseFeeBig.String()) + } + + if nextTip.Cmp(cmath.FromBig(nextTipBig)) != 0 { + t.Fatalf("EffectiveGasTip next incorrect. uint256 %q, big.Int %q, baseFee %q, baseFeeBig %q", nextTip.String(), nextTipBig.String(), baseFee.String(), baseFeeBig.String()) + } + if err != nil || nextErr != nil { t.Errorf("error calculating effective tip") } diff --git a/eth/api_backend.go b/eth/api_backend.go index c33f3cf6f2..2c93e60d87 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -236,11 +236,18 @@ func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri } func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - return b.eth.txPool.AddLocal(signedTx) + err := b.eth.txPool.AddLocal(signedTx) + if err != nil { + if unwrapped := errors.Unwrap(err); unwrapped != nil { + return unwrapped + } + } + + return err } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { - pending := b.eth.txPool.Pending(false) + pending := b.eth.txPool.Pending(context.Background(), false) var txs types.Transactions for _, batch := range pending { txs = append(txs, batch...) diff --git a/eth/bor_checkpoint_verifier.go b/eth/bor_checkpoint_verifier.go index 61e8c382e1..ad81eb6116 100644 --- a/eth/bor_checkpoint_verifier.go +++ b/eth/bor_checkpoint_verifier.go @@ -26,6 +26,7 @@ func newCheckpointVerifier(verifyFn func(ctx context.Context, handler *ethHandle ) // check if we have the checkpoint blocks + //nolint:contextcheck head := handler.ethAPI.BlockNumber() if head < hexutil.Uint64(endBlock) { log.Debug("Head block behind checkpoint block", "head", head, "checkpoint end block", endBlock) diff --git a/eth/handler.go b/eth/handler.go index 8e6d89f9ef..48bdf8eb15 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -17,6 +17,7 @@ package eth import ( + "context" "errors" "math" "math/big" @@ -69,7 +70,7 @@ type txPool interface { // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(enforceTips bool) map[common.Address]types.Transactions + Pending(ctx context.Context, enforceTips bool) map[common.Address]types.Transactions // SubscribeNewTxsEvent should return an event subscription of // NewTxsEvent and send events to the given channel. diff --git a/eth/handler_test.go b/eth/handler_test.go index c6d7811d10..7a14619159 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -17,6 +17,7 @@ package eth import ( + "context" "math/big" "sort" "sync" @@ -92,7 +93,7 @@ func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(enforceTips bool) map[common.Address]types.Transactions { +func (p *testTxPool) Pending(ctx context.Context, enforceTips bool) map[common.Address]types.Transactions { p.lock.RLock() defer p.lock.RUnlock() diff --git a/eth/sync.go b/eth/sync.go index aa79b6181c..377acff95c 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -17,6 +17,7 @@ package eth import ( + "context" "errors" "math/big" "sync/atomic" @@ -44,20 +45,24 @@ func (h *handler) syncTransactions(p *eth.Peer) { // // TODO(karalabe): Figure out if we could get away with random order somehow var txs types.Transactions - pending := h.txpool.Pending(false) + + pending := h.txpool.Pending(context.Background(), false) for _, batch := range pending { txs = append(txs, batch...) } + if len(txs) == 0 { return } // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). + hashes := make([]common.Hash, len(txs)) for i, tx := range txs { hashes[i] = tx.Hash() } + p.AsyncSendPooledTransactionHashes(hashes) } diff --git a/internal/cli/server/pprof/pprof.go b/internal/cli/server/pprof/pprof.go index 44034f3bb8..69056bd0fb 100644 --- a/internal/cli/server/pprof/pprof.go +++ b/internal/cli/server/pprof/pprof.go @@ -61,6 +61,28 @@ func CPUProfile(ctx context.Context, sec int) ([]byte, map[string]string, error) }, nil } +// CPUProfile generates a CPU Profile for a given duration +func CPUProfileWithChannel(done chan bool) ([]byte, map[string]string, error) { + var buf bytes.Buffer + if err := pprof.StartCPUProfile(&buf); err != nil { + return nil, nil, err + } + + select { + case <-done: + case <-time.After(30 * time.Second): + } + + pprof.StopCPUProfile() + + return buf.Bytes(), + map[string]string{ + "X-Content-Type-Options": "nosniff", + "Content-Type": "application/octet-stream", + "Content-Disposition": `attachment; filename="profile"`, + }, nil +} + // Trace runs a trace profile for a given duration func Trace(ctx context.Context, sec int) ([]byte, map[string]string, error) { if sec <= 0 { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 082dfea66f..c1584e5867 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "math/big" + "runtime" "strings" "time" @@ -2229,6 +2230,21 @@ func (api *PrivateDebugAPI) PurgeCheckpointWhitelist() { api.b.PurgeCheckpointWhitelist() } +// GetTraceStack returns the current trace stack +func (api *PrivateDebugAPI) GetTraceStack() string { + buf := make([]byte, 1024) + + for { + n := runtime.Stack(buf, true) + + if n < len(buf) { + return string(buf) + } + + buf = make([]byte, 2*len(buf)) + } +} + // PublicNetAPI offers network related RPC methods type PublicNetAPI struct { net *p2p.Server diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index dcdd5baf23..64ceb5c42e 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -484,6 +484,11 @@ web3._extend({ call: 'debug_purgeCheckpointWhitelist', params: 0, }), + new web3._extend.Method({ + name: 'getTraceStack', + call: 'debug_getTraceStack', + params: 0, + }), ], properties: [] }); diff --git a/les/handler_test.go b/les/handler_test.go index 3ceabdf8ec..af3324b042 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -617,7 +617,7 @@ func testTransactionStatus(t *testing.T, protocol int) { sendRequest(rawPeer.app, GetTxStatusMsg, reqID, []common.Hash{tx.Hash()}) } if err := expectResponse(rawPeer.app, TxStatusMsg, reqID, testBufLimit, []light.TxStatus{expStatus}); err != nil { - t.Errorf("transaction status mismatch") + t.Error("transaction status mismatch", err) } } signer := types.HomesteadSigner{} diff --git a/les/server_requests.go b/les/server_requests.go index 3595a6ab38..b31c11c9d0 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -507,25 +507,39 @@ func handleSendTx(msg Decoder) (serveRequestFn, uint64, uint64, error) { if err := msg.Decode(&r); err != nil { return nil, 0, 0, err } + amount := uint64(len(r.Txs)) + return func(backend serverBackend, p *clientPeer, waitOrStop func() bool) *reply { stats := make([]light.TxStatus, len(r.Txs)) + + var ( + err error + addFn func(transaction *types.Transaction) error + ) + for i, tx := range r.Txs { if i != 0 && !waitOrStop() { return nil } + hash := tx.Hash() stats[i] = txStatus(backend, hash) + if stats[i].Status == core.TxStatusUnknown { - addFn := backend.TxPool().AddRemotes + addFn = backend.TxPool().AddRemote + // Add txs synchronously for testing purpose if backend.AddTxsSync() { - addFn = backend.TxPool().AddRemotesSync + addFn = backend.TxPool().AddRemoteSync } - if errs := addFn([]*types.Transaction{tx}); errs[0] != nil { - stats[i].Error = errs[0].Error() + + if err = addFn(tx); err != nil { + stats[i].Error = err.Error() + continue } + stats[i] = txStatus(backend, hash) } } diff --git a/miner/worker.go b/miner/worker.go index 797e7ea980..0137a74008 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -17,10 +17,15 @@ package miner import ( + "bytes" "context" "errors" "fmt" "math/big" + "os" + "runtime" + "runtime/pprof" + ptrace "runtime/trace" "sync" "sync/atomic" "time" @@ -31,6 +36,7 @@ import ( "go.opentelemetry.io/otel/trace" "github.com/ethereum/go-ethereum/common" + cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/common/tracing" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" @@ -39,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -83,6 +90,12 @@ const ( staleThreshold = 7 ) +// metrics gauge to track total and empty blocks sealed by a miner +var ( + sealedBlocksCounter = metrics.NewRegisteredCounter("worker/sealedBlocks", nil) + sealedEmptyBlocksCounter = metrics.NewRegisteredCounter("worker/sealedEmptyBlocks", nil) +) + // environment is the worker's current environment and holds all // information of the sealing block generation. type environment struct { @@ -257,6 +270,8 @@ type worker struct { skipSealHook func(*task) bool // Method to decide whether skipping the sealing. fullTaskHook func() // Method to call before pushing the full sealing task. resubmitHook func(time.Duration, time.Duration) // Method to call upon updating resubmitting interval. + + profileCount *int32 // Global count for profiling } //nolint:staticcheck @@ -285,6 +300,7 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus resubmitIntervalCh: make(chan time.Duration), resubmitAdjustCh: make(chan *intervalAdjust, resubmitAdjustChanSize), } + worker.profileCount = new(int32) // Subscribe NewTxsEvent for tx pool worker.txsSub = eth.TxPool().SubscribeNewTxsEvent(worker.txsCh) // Subscribe events for blockchain @@ -560,9 +576,11 @@ func (w *worker) mainLoop(ctx context.Context) { for { select { case req := <-w.newWorkCh: + //nolint:contextcheck w.commitWork(req.ctx, req.interrupt, req.noempty, req.timestamp) case req := <-w.getWorkCh: + //nolint:contextcheck block, err := w.generateWork(req.ctx, req.params) if err != nil { req.err = err @@ -622,13 +640,17 @@ func (w *worker) mainLoop(ctx context.Context) { if gp := w.current.gasPool; gp != nil && gp.Gas() < params.TxGas { continue } + txs := make(map[common.Address]types.Transactions) + for _, tx := range ev.Txs { acc, _ := types.Sender(w.current.signer, tx) txs[acc] = append(txs[acc], tx) } - txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs, w.current.header.BaseFee) + + txset := types.NewTransactionsByPriceAndNonce(w.current.signer, txs, cmath.FromBig(w.current.header.BaseFee)) tcount := w.current.tcount + w.commitTransactions(w.current, txset, nil) // Only update the snapshot if any new transactions were added @@ -758,7 +780,7 @@ func (w *worker) resultLoop() { err error ) - tracing.Exec(task.ctx, "resultLoop", func(ctx context.Context, span trace.Span) { + tracing.Exec(task.ctx, "", "resultLoop", func(ctx context.Context, span trace.Span) { for i, taskReceipt := range task.receipts { receipt := new(types.Receipt) receipts[i] = receipt @@ -808,6 +830,12 @@ func (w *worker) resultLoop() { // Broadcast the block and announce chain insertion event w.mux.Post(core.NewMinedBlockEvent{Block: block}) + sealedBlocksCounter.Inc(1) + + if block.Transactions().Len() == 0 { + sealedEmptyBlocksCounter.Inc(1) + } + // Insert the block into the set of pending ones to resultLoop for confirmations w.unconfirmed.Insert(block.NumberU64(), block.Hash()) case <-w.exitCh: @@ -965,7 +993,10 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP // Start executing the transaction env.state.Prepare(tx.Hash(), env.tcount) + start := time.Now() + logs, err := w.commitTransaction(env, tx) + switch { case errors.Is(err, core.ErrGasLimitReached): // Pop the current out-of-gas transaction without shifting in the next from the account @@ -987,6 +1018,7 @@ func (w *worker) commitTransactions(env *environment, txs *types.TransactionsByP coalescedLogs = append(coalescedLogs, logs...) env.tcount++ txs.Shift() + log.Info("Committed new tx", "tx hash", tx.Hash(), "from", from, "to", tx.To(), "nonce", tx.Nonce(), "gas", tx.Gas(), "gasPrice", tx.GasPrice(), "value", tx.Value(), "time spent", time.Since(start)) case errors.Is(err, core.ErrTxTypeNotSupported): // Pop the unsupported transaction without shifting in the next from the account @@ -1077,7 +1109,7 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { } // Set baseFee and GasLimit if we are on an EIP-1559 chain if w.chainConfig.IsLondon(header.Number) { - header.BaseFee = misc.CalcBaseFee(w.chainConfig, parent.Header()) + header.BaseFee = misc.CalcBaseFeeUint(w.chainConfig, parent.Header()).ToBig() if !w.chainConfig.IsLondon(parent.Number()) { parentGasLimit := parent.GasLimit() * params.ElasticityMultiplier header.GasLimit = core.CalcGasLimit(parentGasLimit, w.config.GasCeil) @@ -1117,9 +1149,75 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) { return env, nil } +func startProfiler(profile string, filepath string, number uint64) (func() error, error) { + var ( + buf bytes.Buffer + err error + ) + + closeFn := func() {} + + switch profile { + case "cpu": + err = pprof.StartCPUProfile(&buf) + + if err == nil { + closeFn = func() { + pprof.StopCPUProfile() + } + } + case "trace": + err = ptrace.Start(&buf) + + if err == nil { + closeFn = func() { + ptrace.Stop() + } + } + case "heap": + runtime.GC() + + err = pprof.WriteHeapProfile(&buf) + default: + log.Info("Incorrect profile name") + } + + if err != nil { + return func() error { + closeFn() + return nil + }, err + } + + closeFnNew := func() error { + var err error + + closeFn() + + if buf.Len() == 0 { + return nil + } + + f, err := os.Create(filepath + "/" + profile + "-" + fmt.Sprint(number) + ".prof") + if err != nil { + return err + } + + defer f.Close() + + _, err = f.Write(buf.Bytes()) + + return err + } + + return closeFnNew, nil +} + // fillTransactions retrieves the pending transactions from the txpool and fills them // into the given sealing block. The transaction selection and ordering strategy can // be customized with the plugin in the future. +// +//nolint:gocognit func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *environment) { ctx, span := tracing.StartSpan(ctx, "fillTransactions") defer tracing.EndSpan(span) @@ -1134,10 +1232,76 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en remoteTxs map[common.Address]types.Transactions ) - tracing.Exec(ctx, "worker.SplittingTransactions", func(ctx context.Context, span trace.Span) { - pending := w.eth.TxPool().Pending(true) + // TODO: move to config or RPC + const profiling = false + + if profiling { + doneCh := make(chan struct{}) + + defer func() { + close(doneCh) + }() + + go func(number uint64) { + closeFn := func() error { + return nil + } + + for { + select { + case <-time.After(150 * time.Millisecond): + // Check if we've not crossed limit + if attempt := atomic.AddInt32(w.profileCount, 1); attempt >= 10 { + log.Info("Completed profiling", "attempt", attempt) + + return + } + + log.Info("Starting profiling in fill transactions", "number", number) + + dir, err := os.MkdirTemp("", fmt.Sprintf("bor-traces-%s-", time.Now().UTC().Format("2006-01-02-150405Z"))) + if err != nil { + log.Error("Error in profiling", "path", dir, "number", number, "err", err) + return + } + + // grab the cpu profile + closeFnInternal, err := startProfiler("cpu", dir, number) + if err != nil { + log.Error("Error in profiling", "path", dir, "number", number, "err", err) + return + } + + closeFn = func() error { + err := closeFnInternal() + + log.Info("Completed profiling", "path", dir, "number", number, "error", err) + + return nil + } + + case <-doneCh: + err := closeFn() + + if err != nil { + log.Info("closing fillTransactions", "number", number, "error", err) + } + + return + } + } + }(env.header.Number.Uint64()) + } + + tracing.Exec(ctx, "", "worker.SplittingTransactions", func(ctx context.Context, span trace.Span) { + + prePendingTime := time.Now() + + pending := w.eth.TxPool().Pending(ctx, true) remoteTxs = pending + postPendingTime := time.Now() + for _, account := range w.eth.TxPool().Locals() { if txs := remoteTxs[account]; len(txs) > 0 { delete(remoteTxs, account) @@ -1145,6 +1309,8 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en } } + postLocalsTime := time.Now() + localTxsCount = len(localTxs) remoteTxsCount = len(remoteTxs) @@ -1152,6 +1318,8 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en span, attribute.Int("len of local txs", localTxsCount), attribute.Int("len of remote txs", remoteTxsCount), + attribute.String("time taken by Pending()", fmt.Sprintf("%v", postPendingTime.Sub(prePendingTime))), + attribute.String("time taken by Locals()", fmt.Sprintf("%v", postLocalsTime.Sub(postPendingTime))), ) }) @@ -1164,8 +1332,8 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en if localTxsCount > 0 { var txs *types.TransactionsByPriceAndNonce - tracing.Exec(ctx, "worker.LocalTransactionsByPriceAndNonce", func(ctx context.Context, span trace.Span) { - txs = types.NewTransactionsByPriceAndNonce(env.signer, localTxs, env.header.BaseFee) + tracing.Exec(ctx, "", "worker.LocalTransactionsByPriceAndNonce", func(ctx context.Context, span trace.Span) { + txs = types.NewTransactionsByPriceAndNonce(env.signer, localTxs, cmath.FromBig(env.header.BaseFee)) tracing.SetAttributes( span, @@ -1173,7 +1341,7 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en ) }) - tracing.Exec(ctx, "worker.LocalCommitTransactions", func(ctx context.Context, span trace.Span) { + tracing.Exec(ctx, "", "worker.LocalCommitTransactions", func(ctx context.Context, span trace.Span) { committed = w.commitTransactions(env, txs, interrupt) }) @@ -1187,8 +1355,8 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en if remoteTxsCount > 0 { var txs *types.TransactionsByPriceAndNonce - tracing.Exec(ctx, "worker.RemoteTransactionsByPriceAndNonce", func(ctx context.Context, span trace.Span) { - txs = types.NewTransactionsByPriceAndNonce(env.signer, remoteTxs, env.header.BaseFee) + tracing.Exec(ctx, "", "worker.RemoteTransactionsByPriceAndNonce", func(ctx context.Context, span trace.Span) { + txs = types.NewTransactionsByPriceAndNonce(env.signer, remoteTxs, cmath.FromBig(env.header.BaseFee)) tracing.SetAttributes( span, @@ -1196,7 +1364,7 @@ func (w *worker) fillTransactions(ctx context.Context, interrupt *int32, env *en ) }) - tracing.Exec(ctx, "worker.RemoteCommitTransactions", func(ctx context.Context, span trace.Span) { + tracing.Exec(ctx, "", "worker.RemoteCommitTransactions", func(ctx context.Context, span trace.Span) { committed = w.commitTransactions(env, txs, interrupt) }) @@ -1237,7 +1405,7 @@ func (w *worker) commitWork(ctx context.Context, interrupt *int32, noempty bool, err error ) - tracing.Exec(ctx, "worker.prepareWork", func(ctx context.Context, span trace.Span) { + tracing.Exec(ctx, "", "worker.prepareWork", func(ctx context.Context, span trace.Span) { // Set the coinbase if the worker is running or it's required var coinbase common.Address if w.isRunning() { diff --git a/tests/init_test.go b/tests/init_test.go index 1c6841e030..5e32f20abf 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -141,9 +141,6 @@ func (tm *testMatcher) findSkip(name string) (reason string, skipload bool) { isWin32 := runtime.GOARCH == "386" && runtime.GOOS == "windows" for _, re := range tm.slowpat { if re.MatchString(name) { - if testing.Short() { - return "skipped in -short mode", false - } if isWin32 { return "skipped on 32bit windows", false } From 243d231fe45bc02f33678bb4f69e941167d7f466 Mon Sep 17 00:00:00 2001 From: builder90210 Date: Thu, 8 Dec 2022 00:30:10 -0800 Subject: [PATCH 65/70] Reduce txArriveTimeout to 100ms --- eth/fetcher/tx_fetcher.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index b10c0db9ee..8b97746b14 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -55,11 +55,11 @@ const ( // txArriveTimeout is the time allowance before an announced transaction is // explicitly requested. - txArriveTimeout = 500 * time.Millisecond + txArriveTimeout = 100 * time.Millisecond // txGatherSlack is the interval used to collate almost-expired announces // with network fetches. - txGatherSlack = 100 * time.Millisecond + txGatherSlack = 20 * time.Millisecond ) var ( From 9cba79edee6d859d8bdddb9e077ba821f435495d Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 19 Dec 2022 13:57:18 +0530 Subject: [PATCH 66/70] init : remove exit on keystore err --- internal/cli/server/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index b8310aea6f..f1056ca769 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -722,7 +722,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* for i, account := range c.Accounts.Unlock { err = ks.Unlock(accounts.Account{Address: common.HexToAddress(account)}, passwords[i]) if err != nil { - return nil, fmt.Errorf("could not unlock an account %q", account) + log.Warn("Could not unlock account", "account", account, "err", err) } } } From 43cafc0415ad23d75a8d50667a14cc3155aef7f2 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 19 Dec 2022 18:06:16 +0530 Subject: [PATCH 67/70] add : multiple keystore tolerance --- internal/cli/server/config.go | 61 ++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index f1056ca769..7dfb0df6c7 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/eth/downloader" @@ -720,10 +721,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* } for i, account := range c.Accounts.Unlock { - err = ks.Unlock(accounts.Account{Address: common.HexToAddress(account)}, passwords[i]) - if err != nil { - log.Warn("Could not unlock account", "account", account, "err", err) - } + unlockAccount(ks, account, i, passwords) } } @@ -904,6 +902,61 @@ var ( gitDate = "" // Git commit date YYYYMMDD of the release (set via linker flags) ) +// tries unlocking the specified account a few times. +func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) { + account, err := utils.MakeAddress(ks, address) + if err != nil { + utils.Fatalf("Could not list accounts: %v", err) + } + for trials := 0; trials < 3; trials++ { + prompt := fmt.Sprintf("Unlocking account %s | Attempt %d/%d", address, trials+1, 3) + password := utils.GetPassPhraseWithList(prompt, false, i, passwords) + err = ks.Unlock(account, password) + if err == nil { + log.Info("Unlocked account", "address", account.Address.Hex()) + return account, password + } + if err, ok := err.(*keystore.AmbiguousAddrError); ok { + log.Info("Unlocked account", "address", account.Address.Hex()) + return ambiguousAddrRecovery(ks, err, password), password + } + if err != keystore.ErrDecrypt { + // No need to prompt again if the error is not decryption-related. + break + } + } + // All trials expended to unlock account, bail out + utils.Fatalf("Failed to unlock account %s (%v)", address, err) + + return accounts.Account{}, "" +} + +func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrError, auth string) accounts.Account { + fmt.Printf("Multiple key files exist for address %x:\n", err.Addr) + for _, a := range err.Matches { + fmt.Println(" ", a.URL) + } + fmt.Println("Testing your password against all of them...") + var match *accounts.Account + for _, a := range err.Matches { + if err := ks.Unlock(a, auth); err == nil { + match = &a + break + } + } + if match == nil { + utils.Fatalf("None of the listed files could be unlocked.") + } + fmt.Printf("Your password unlocked %s\n", match.URL) + fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") + for _, a := range err.Matches { + if a != *match { + fmt.Println(" ", a.URL) + } + } + return *match +} + func (c *Config) buildNode() (*node.Config, error) { ipcPath := "" if !c.JsonRPC.IPCDisable { From 828801f9d8d0c80ba1bc56ed223975406b9896c5 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Mon, 19 Dec 2022 18:36:10 +0530 Subject: [PATCH 68/70] lint : fix linters --- internal/cli/server/config.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 7dfb0df6c7..1d011cc665 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -905,21 +905,26 @@ var ( // tries unlocking the specified account a few times. func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []string) (accounts.Account, string) { account, err := utils.MakeAddress(ks, address) + if err != nil { utils.Fatalf("Could not list accounts: %v", err) } + for trials := 0; trials < 3; trials++ { prompt := fmt.Sprintf("Unlocking account %s | Attempt %d/%d", address, trials+1, 3) password := utils.GetPassPhraseWithList(prompt, false, i, passwords) err = ks.Unlock(account, password) + if err == nil { log.Info("Unlocked account", "address", account.Address.Hex()) return account, password } + if err, ok := err.(*keystore.AmbiguousAddrError); ok { log.Info("Unlocked account", "address", account.Address.Hex()) return ambiguousAddrRecovery(ks, err, password), password } + if err != keystore.ErrDecrypt { // No need to prompt again if the error is not decryption-related. break @@ -933,27 +938,36 @@ func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []str func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrError, auth string) accounts.Account { fmt.Printf("Multiple key files exist for address %x:\n", err.Addr) + for _, a := range err.Matches { fmt.Println(" ", a.URL) } + fmt.Println("Testing your password against all of them...") + var match *accounts.Account + for _, a := range err.Matches { if err := ks.Unlock(a, auth); err == nil { + // nolint: gosec, exportloopref match = &a break } } + if match == nil { utils.Fatalf("None of the listed files could be unlocked.") } + fmt.Printf("Your password unlocked %s\n", match.URL) fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") + for _, a := range err.Matches { if a != *match { fmt.Println(" ", a.URL) } } + return *match } From 513127cd8f881940b9be8acfaea8f1f573dc7873 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Tue, 20 Dec 2022 10:54:03 +0530 Subject: [PATCH 69/70] chg : use standard logging --- internal/cli/server/config.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 1d011cc665..83971cf217 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -937,13 +937,13 @@ func unlockAccount(ks *keystore.KeyStore, address string, i int, passwords []str } func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrError, auth string) accounts.Account { - fmt.Printf("Multiple key files exist for address %x:\n", err.Addr) + log.Warn("Multiple key files exist for", "address", err.Addr) for _, a := range err.Matches { - fmt.Println(" ", a.URL) + log.Info("Multiple keys", "file", a.URL) } - fmt.Println("Testing your password against all of them...") + log.Info("Testing your password against all of them...") var match *accounts.Account @@ -959,8 +959,8 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr utils.Fatalf("None of the listed files could be unlocked.") } - fmt.Printf("Your password unlocked %s\n", match.URL) - fmt.Println("In order to avoid this warning, you need to remove the following duplicate key files:") + log.Info("Your password unlocked", "key", match.URL) + log.Warn("In order to avoid this warning, you need to remove the following duplicate key files:") for _, a := range err.Matches { if a != *match { From 7b52c93c1bfcd4dc2f40f68bfd3e8a5c4c1d6347 Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Tue, 20 Dec 2022 11:39:11 +0530 Subject: [PATCH 70/70] chg : logging strings --- internal/cli/server/config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index 83971cf217..34c17b3f7d 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -940,7 +940,7 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr log.Warn("Multiple key files exist for", "address", err.Addr) for _, a := range err.Matches { - log.Info("Multiple keys", "file", a.URL) + log.Info("Multiple keys", "file", a.URL.String()) } log.Info("Testing your password against all of them...") @@ -959,12 +959,12 @@ func ambiguousAddrRecovery(ks *keystore.KeyStore, err *keystore.AmbiguousAddrErr utils.Fatalf("None of the listed files could be unlocked.") } - log.Info("Your password unlocked", "key", match.URL) + log.Info("Your password unlocked", "key", match.URL.String()) log.Warn("In order to avoid this warning, you need to remove the following duplicate key files:") for _, a := range err.Matches { if a != *match { - fmt.Println(" ", a.URL) + log.Warn("Duplicate", "key", a.URL.String()) } }