Add support for new flags in new config.toml, which were present in old config.toml (#612)

* added HTTPTimeouts, and TrieTimeout flag in new tol, from old toml

* added RAW fields for these time.Duration flags

* updated the conversion script to support these extra 4 flags

* removed hcl and json config tests as we are only supporting toml config files

* updated toml files with cache.timeout field

* updated toml files with jsonrpc.timeouts field

* tests/bor: expect a call for latest checkpoint

* tests/bor: expect a call for latest checkpoint

* packaging/templates: update cache values for archive nodes

Co-authored-by: Manav Darji <manavdarji.india@gmail.com>
This commit is contained in:
Pratik Patil 2022-12-05 19:21:33 +05:30 committed by GitHub
parent dd64fa160c
commit 57075d000d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 233 additions and 99 deletions

View file

@ -61,32 +61,36 @@ syncmode = "full"
# [jsonrpc]
# ipcdisable = false
# ipcpath = ""
# gascap = 50000000
# txfeecap = 5.0
# [jsonrpc.http]
# enabled = false
# port = 8545
# prefix = ""
# host = "localhost"
# api = ["eth", "net", "web3", "txpool", "bor"]
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.ws]
# enabled = false
# port = 8546
# prefix = ""
# host = "localhost"
# api = ["web3", "net"]
# origins = ["*"]
# [jsonrpc.graphql]
# enabled = false
# port = 0
# prefix = ""
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# ipcdisable = false
# ipcpath = ""
# gascap = 50000000
# txfeecap = 5.0
# [jsonrpc.http]
# enabled = false
# port = 8545
# prefix = ""
# host = "localhost"
# api = ["eth", "net", "web3", "txpool", "bor"]
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.ws]
# enabled = false
# port = 8546
# prefix = ""
# host = "localhost"
# api = ["web3", "net"]
# origins = ["*"]
# [jsonrpc.graphql]
# enabled = false
# port = 0
# prefix = ""
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
[gpo]
# blocks = 20
@ -122,6 +126,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
[accounts]
# allow-insecure-unlock = true

View file

@ -86,6 +86,10 @@ ethstats = "" # Reporting URL of a ethstats service (nodename:sec
host = "" #
vhosts = ["localhost"] # Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard.
corsdomain = ["localhost"] # Comma separated list of domains from which to accept cross origin requests (browser enforced)
[jsonrpc.timeouts]
read = "30s"
write = "30s"
idle = "2m0s"
[gpo]
blocks = 20 # Number of recent blocks to check for gas prices
@ -126,6 +130,7 @@ ethstats = "" # Reporting URL of a ethstats service (nodename:sec
preimages = false # Enable recording the SHA3/keccak preimages of trie keys
txlookuplimit = 2350000 # Number of recent blocks to maintain transactions index for (default = about 56 days, 0 = entire chain)
triesinmemory = 128 # Number of block states (tries) to keep in memory
timeout = "1h0m0s" # Time after which the Merkle Patricia Trie is stored to disc from memory
[accounts]
unlock = [] # Comma separated list of accounts to unlock

View file

@ -52,12 +52,16 @@ func (c *DumpconfigCommand) Run(args []string) int {
userConfig := command.GetConfig()
// convert the big.Int and time.Duration fields to their corresponding Raw fields
userConfig.JsonRPC.HttpTimeout.ReadTimeoutRaw = userConfig.JsonRPC.HttpTimeout.ReadTimeout.String()
userConfig.JsonRPC.HttpTimeout.WriteTimeoutRaw = userConfig.JsonRPC.HttpTimeout.WriteTimeout.String()
userConfig.JsonRPC.HttpTimeout.IdleTimeoutRaw = userConfig.JsonRPC.HttpTimeout.IdleTimeout.String()
userConfig.TxPool.RejournalRaw = userConfig.TxPool.Rejournal.String()
userConfig.TxPool.LifeTimeRaw = userConfig.TxPool.LifeTime.String()
userConfig.Sealer.GasPriceRaw = userConfig.Sealer.GasPrice.String()
userConfig.Gpo.MaxPriceRaw = userConfig.Gpo.MaxPrice.String()
userConfig.Gpo.IgnorePriceRaw = userConfig.Gpo.IgnorePrice.String()
userConfig.Cache.RejournalRaw = userConfig.Cache.Rejournal.String()
userConfig.Cache.TrieTimeoutRaw = userConfig.Cache.TrieTimeout.String()
if err := toml.NewEncoder(os.Stdout).Encode(userConfig); err != nil {
c.UI.Error(err.Error())

View file

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/p2p/nat"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
type Config struct {
@ -238,6 +239,8 @@ type JsonRPCConfig struct {
// Graphql has the json-rpc graphql related settings
Graphql *APIConfig `hcl:"graphql,block" toml:"graphql,block"`
HttpTimeout *HttpTimeouts `hcl:"timeouts,block" toml:"timeouts,block"`
}
type GRPCConfig struct {
@ -271,6 +274,33 @@ type APIConfig struct {
Origins []string `hcl:"origins,optional" toml:"origins,optional"`
}
// Used from rpc.HTTPTimeouts
type HttpTimeouts struct {
// ReadTimeout is the maximum duration for reading the entire
// request, including the body.
//
// Because ReadTimeout does not let Handlers make per-request
// decisions on each request body's acceptable deadline or
// upload rate, most users will prefer to use
// ReadHeaderTimeout. It is valid to use them both.
ReadTimeout time.Duration `hcl:"-,optional" toml:"-"`
ReadTimeoutRaw string `hcl:"read,optional" toml:"read,optional"`
// WriteTimeout is the maximum duration before timing out
// writes of the response. It is reset whenever a new
// request's header is read. Like ReadTimeout, it does not
// let Handlers make decisions on a per-request basis.
WriteTimeout time.Duration `hcl:"-,optional" toml:"-"`
WriteTimeoutRaw string `hcl:"write,optional" toml:"write,optional"`
// IdleTimeout is the maximum amount of time to wait for the
// next request when keep-alives are enabled. If IdleTimeout
// is zero, the value of ReadTimeout is used. If both are
// zero, ReadHeaderTimeout is used.
IdleTimeout time.Duration `hcl:"-,optional" toml:"-"`
IdleTimeoutRaw string `hcl:"idle,optional" toml:"idle,optional"`
}
type GpoConfig struct {
// Blocks is the number of blocks to track to compute the price oracle
Blocks uint64 `hcl:"blocks,optional" toml:"blocks,optional"`
@ -367,6 +397,10 @@ type CacheConfig struct {
// TxLookupLimit sets the maximum number of blocks from head whose tx indices are reserved.
TxLookupLimit uint64 `hcl:"txlookuplimit,optional" toml:"txlookuplimit,optional"`
// Time after which the Merkle Patricia Trie is stored to disc from memory
TrieTimeout time.Duration `hcl:"-,optional" toml:"-"`
TrieTimeoutRaw string `hcl:"timeout,optional" toml:"timeout,optional"`
}
type AccountsConfig struct {
@ -480,6 +514,11 @@ func DefaultConfig() *Config {
Cors: []string{"localhost"},
VHost: []string{"localhost"},
},
HttpTimeout: &HttpTimeouts{
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 120 * time.Second,
},
},
Ethstats: "",
Telemetry: &TelemetryConfig{
@ -511,6 +550,7 @@ func DefaultConfig() *Config {
NoPrefetch: false,
Preimages: false,
TxLookupLimit: 2350000,
TrieTimeout: 60 * time.Minute,
},
Accounts: &AccountsConfig{
Unlock: []string{},
@ -570,9 +610,13 @@ func (c *Config) fillTimeDurations() error {
td *time.Duration
str *string
}{
{"jsonrpc.timeouts.read", &c.JsonRPC.HttpTimeout.ReadTimeout, &c.JsonRPC.HttpTimeout.ReadTimeoutRaw},
{"jsonrpc.timeouts.write", &c.JsonRPC.HttpTimeout.WriteTimeout, &c.JsonRPC.HttpTimeout.WriteTimeoutRaw},
{"jsonrpc.timeouts.idle", &c.JsonRPC.HttpTimeout.IdleTimeout, &c.JsonRPC.HttpTimeout.IdleTimeoutRaw},
{"txpool.lifetime", &c.TxPool.LifeTime, &c.TxPool.LifeTimeRaw},
{"txpool.rejournal", &c.TxPool.Rejournal, &c.TxPool.RejournalRaw},
{"cache.rejournal", &c.Cache.Rejournal, &c.Cache.RejournalRaw},
{"cache.timeout", &c.Cache.TrieTimeout, &c.Cache.TrieTimeoutRaw},
}
for _, x := range tds {
@ -830,6 +874,7 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (*
n.NoPrefetch = c.Cache.NoPrefetch
n.Preimages = c.Cache.Preimages
n.TxLookupLimit = c.Cache.TxLookupLimit
n.TrieTimeout = c.Cache.TrieTimeout
}
n.RPCGasCap = c.JsonRPC.GasCap
@ -928,6 +973,11 @@ func (c *Config) buildNode() (*node.Config, error) {
WSPathPrefix: c.JsonRPC.Ws.Prefix,
GraphQLCors: c.JsonRPC.Graphql.Cors,
GraphQLVirtualHosts: c.JsonRPC.Graphql.VHost,
HTTPTimeouts: rpc.HTTPTimeouts{
ReadTimeout: c.JsonRPC.HttpTimeout.ReadTimeout,
WriteTimeout: c.JsonRPC.HttpTimeout.WriteTimeout,
IdleTimeout: c.JsonRPC.HttpTimeout.IdleTimeout,
},
}
// dev mode

View file

@ -1,7 +1,6 @@
package server
import (
"math/big"
"testing"
"time"
@ -101,38 +100,6 @@ func TestDefaultDatatypeOverride(t *testing.T) {
assert.Equal(t, c0, expected)
}
func TestConfigLoadFile(t *testing.T) {
readFile := func(path string) {
config, err := readConfigFile(path)
assert.NoError(t, err)
assert.Equal(t, config, &Config{
DataDir: "./data",
P2P: &P2PConfig{
MaxPeers: 30,
},
TxPool: &TxPoolConfig{
LifeTime: 1 * time.Second,
},
Gpo: &GpoConfig{
MaxPrice: big.NewInt(100),
},
Sealer: &SealerConfig{},
Cache: &CacheConfig{},
})
}
// read file in hcl format
t.Run("hcl", func(t *testing.T) {
readFile("./testdata/test.hcl")
})
// read file in json format
t.Run("json", func(t *testing.T) {
readFile("./testdata/test.json")
})
}
var dummyEnodeAddr = "enode://0cb82b395094ee4a2915e9714894627de9ed8498fb881cec6db7c65e8b9a5bd7f2f25cc84e71e89d0947e51c76e85d0847de848c7782b13c0255247a6758178c@44.232.55.71:30303"
func TestConfigBootnodesDefault(t *testing.T) {

View file

@ -1,13 +0,0 @@
datadir = "./data"
p2p {
maxpeers = 30
}
txpool {
lifetime = "1s"
}
gpo {
maxprice = "100"
}

View file

@ -1,12 +0,0 @@
{
"datadir": "./data",
"p2p": {
"maxpeers": 30
},
"txpool": {
"lifetime": "1s"
},
"gpo": {
"maxprice": "100"
}
}

View file

@ -79,6 +79,10 @@ gcmode = "archive"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
[gpo]
# blocks = 20
@ -105,15 +109,16 @@ gcmode = "archive"
[cache]
cache = 4096
# gc = 25
# snapshot = 10
gc = 0
snapshot = 20
# database = 50
# trie = 15
trie = 30
# journal = "triecache"
# rejournal = "1h0m0s"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
# [accounts]
# unlock = []

View file

@ -79,6 +79,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
[gpo]
# blocks = 20
@ -114,6 +118,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
# [accounts]
# unlock = []

View file

@ -81,6 +81,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
[gpo]
# blocks = 20
@ -116,6 +120,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
[accounts]
allow-insecure-unlock = true

View file

@ -81,6 +81,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
[gpo]
# blocks = 20
@ -116,6 +120,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
[accounts]
allow-insecure-unlock = true

View file

@ -79,6 +79,10 @@ gcmode = "archive"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
# [gpo]
# blocks = 20
@ -103,17 +107,18 @@ gcmode = "archive"
# organization = ""
# [telemetry.influx.tags]
# [cache]
[cache]
# cache = 1024
# gc = 25
# snapshot = 10
gc = 0
snapshot = 20
# database = 50
# trie = 15
trie = 30
# journal = "triecache"
# rejournal = "1h0m0s"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
# [accounts]
# unlock = []

View file

@ -79,6 +79,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
# [gpo]
# blocks = 20
@ -114,6 +118,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
# [accounts]
# unlock = []

View file

@ -81,6 +81,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
# [gpo]
# blocks = 20
@ -116,6 +120,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
[accounts]
allow-insecure-unlock = true

View file

@ -81,6 +81,10 @@ syncmode = "full"
# host = ""
# vhosts = ["*"]
# corsdomain = ["*"]
# [jsonrpc.timeouts]
# read = "30s"
# write = "30s"
# idle = "2m0s"
# [gpo]
# blocks = 20
@ -116,6 +120,7 @@ syncmode = "full"
# noprefetch = false
# preimages = false
# txlookuplimit = 2350000
# timeout = "1h0m0s"
[accounts]
allow-insecure-unlock = true

View file

@ -357,6 +357,34 @@ func writeTempStaticTrustedTOML(path string) {
log.Fatal(err)
}
}
if data.Has("Node.HTTPTimeouts.ReadTimeout") {
err = os.WriteFile("./tempHTTPTimeoutsReadTimeout.toml", []byte(data.Get("Node.HTTPTimeouts.ReadTimeout").(string)), 0600)
if err != nil {
log.Fatal(err)
}
}
if data.Has("Node.HTTPTimeouts.WriteTimeout") {
err = os.WriteFile("./tempHTTPTimeoutsWriteTimeout.toml", []byte(data.Get("Node.HTTPTimeouts.WriteTimeout").(string)), 0600)
if err != nil {
log.Fatal(err)
}
}
if data.Has("Node.HTTPTimeouts.IdleTimeout") {
err = os.WriteFile("./tempHTTPTimeoutsIdleTimeout.toml", []byte(data.Get("Node.HTTPTimeouts.IdleTimeout").(string)), 0600)
if err != nil {
log.Fatal(err)
}
}
if data.Has("Eth.TrieTimeout") {
err = os.WriteFile("./tempHTTPTimeoutsTrieTimeout.toml", []byte(data.Get("Eth.TrieTimeout").(string)), 0600)
if err != nil {
log.Fatal(err)
}
}
}
func getStaticTrustedNodes(args []string) {
@ -574,7 +602,7 @@ func commentFlags(path string, updatedArgs []string) {
flag = strconv.Itoa(passwordFlag) + "-" + flag
}
if flag != "static-nodes" && flag != "trusted-nodes" {
if flag != "static-nodes" && flag != "trusted-nodes" && flag != "read" && flag != "write" && flag != "idle" && flag != "timeout" {
flag = nameTagMap[flag]
tempFlag := false

View file

@ -112,6 +112,54 @@ else
echo "neither JSON nor TOML TrustedNodes found"
fi
if [[ -f ./tempHTTPTimeoutsReadTimeout.toml ]]
then
echo "HTTPTimeouts.ReadTimeout found"
read=$(head -1 ./tempHTTPTimeoutsReadTimeout.toml)
shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then
sed -i '' "s%read = \"30s\"%read = \"${read}\"%" $confPath
else
sed -i "s%read = \"30s\"%read = \"${read}\"%" $confPath
fi
rm ./tempHTTPTimeoutsReadTimeout.toml
fi
if [[ -f ./tempHTTPTimeoutsWriteTimeout.toml ]]
then
echo "HTTPTimeouts.WriteTimeout found"
write=$(head -1 ./tempHTTPTimeoutsWriteTimeout.toml)
shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then
sed -i '' "s%write = \"30s\"%write = \"${write}\"%" $confPath
else
sed -i "s%write = \"30s\"%write = \"${write}\"%" $confPath
fi
rm ./tempHTTPTimeoutsWriteTimeout.toml
fi
if [[ -f ./tempHTTPTimeoutsIdleTimeout.toml ]]
then
echo "HTTPTimeouts.IdleTimeout found"
idle=$(head -1 ./tempHTTPTimeoutsIdleTimeout.toml)
shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then
sed -i '' "s%idle = \"2m0s\"%idle = \"${idle}\"%" $confPath
else
sed -i "s%idle = \"2m0s\"%idle = \"${idle}\"%" $confPath
fi
rm ./tempHTTPTimeoutsIdleTimeout.toml
fi
if [[ -f ./tempHTTPTimeoutsTrieTimeout.toml ]]
then
echo "Eth.TrieTimeout found"
timeout=$(head -1 ./tempHTTPTimeoutsTrieTimeout.toml)
shopt -s nocasematch; if [[ "$OS" == "darwin"* ]]; then
sed -i '' "s%timeout = \"1h0m0s\"%timeout = \"${timeout}\"%" $confPath
else
sed -i "s%timeout = \"1h0m0s\"%timeout = \"${timeout}\"%" $confPath
fi
rm ./tempHTTPTimeoutsTrieTimeout.toml
fi
printf "\n"
# comment flags in $configPath that were not passed through $startPath

View file

@ -101,6 +101,11 @@ func TestFetchStateSyncEvents(t *testing.T) {
h := mocks.NewMockIHeimdallClient(ctrl)
h.EXPECT().Close().AnyTimes()
h.EXPECT().Span(uint64(1)).Return(&res.Result, nil).AnyTimes()
h.EXPECT().FetchLatestCheckpoint().Return(&checkpoint.Checkpoint{
StartBlock: big.NewInt(1),
EndBlock: big.NewInt(2),
RootHash: common.Hash{},
}, nil).AnyTimes()
// B.2 Mock State Sync events
fromID := uint64(1)
@ -136,6 +141,11 @@ func TestFetchStateSyncEvents_2(t *testing.T) {
h := mocks.NewMockIHeimdallClient(ctrl)
h.EXPECT().Close().AnyTimes()
h.EXPECT().Span(uint64(1)).Return(&res.Result, nil).AnyTimes()
h.EXPECT().FetchLatestCheckpoint().Return(&checkpoint.Checkpoint{
StartBlock: big.NewInt(1),
EndBlock: big.NewInt(2),
RootHash: common.Hash{},
}, nil).AnyTimes()
// Mock State Sync events
// at # sprintSize, events are fetched for [fromID, (block-sprint).Time)
@ -287,6 +297,8 @@ func getMockedHeimdallClient(t *testing.T) (*mocks.MockIHeimdallClient, *span.He
h.EXPECT().StateSyncEvents(gomock.Any(), gomock.Any()).
Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
// h.EXPECT().FetchLatestCheckpoint().Return([]*clerk.EventRecordWithTime{getSampleEventRecord(t)}, nil).AnyTimes()
return h, heimdallSpan, ctrl
}