From cfb7a1cd1c62d3fcd250606a8065be910a467b40 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Thu, 22 Sep 2022 09:37:27 +0530 Subject: [PATCH 01/10] new: unskip failing tests --- tests/block_test.go | 6 +++--- tests/state_test.go | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 6bdb98679f..b3eb71ed28 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -51,9 +51,9 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) // FIXME POS-618 - bt.skipLoad(`.*ValidBlocks*`) - bt.skipLoad(`.*InvalidBlocks*`) - bt.skipLoad(`.*TransitionTests*`) + //bt.skipLoad(`.*ValidBlocks*`) + //bt.skipLoad(`.*InvalidBlocks*`) + //bt.skipLoad(`.*TransitionTests*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, test.Run(false)); err != nil { diff --git a/tests/state_test.go b/tests/state_test.go index 1e7171ee20..eeb19570df 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -59,7 +59,7 @@ func TestState(t *testing.T) { st.skipLoad(`^stStaticCall/static_Call1MB`) // FIXME POS-618 - st.skipLoad(`.*micro/*`) + /*st.skipLoad(`.*micro/*`) st.skipLoad(`.*main/*`) st.skipLoad(`.*stSStoreTest*`) st.skipLoad(`.*stReturnDataTest*`) @@ -115,6 +115,7 @@ func TestState(t *testing.T) { st.skipLoad(`.*stBadOpcode*`) st.skipLoad(`.*stStackTests*`) st.skipLoad(`.*stAttackTest*`) + */ // Broken tests: // Expected failures: From f51eae03ac30e66783c3326b46b8ba45a82dd357 Mon Sep 17 00:00:00 2001 From: Arpit Temani Date: Thu, 22 Sep 2022 13:22:20 +0530 Subject: [PATCH 02/10] disable macos tests --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a85cb300d2..a38b1f5295 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: if: (github.event.action != 'closed' || github.event.pull_request.merged == true) strategy: matrix: - os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments + os: [ ubuntu-20.04 ] # list of os: https://github.com/actions/virtual-environments runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -63,7 +63,7 @@ jobs: - name: test-integration run: make test-integration - + - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: From aa2ce7afb0c3565018ecf425689affc312db8c7d Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Thu, 22 Sep 2022 18:00:23 +0530 Subject: [PATCH 03/10] fix: add bor config --- tests/block_test.go | 4 ++-- tests/init.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index b3eb71ed28..a1d700d02e 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -51,8 +51,8 @@ func TestBlockchain(t *testing.T) { bt.skipLoad(`.*randomStatetest94.json.*`) // FIXME POS-618 - //bt.skipLoad(`.*ValidBlocks*`) - //bt.skipLoad(`.*InvalidBlocks*`) + bt.skipLoad(`.*ValidBlocks*`) + bt.skipLoad(`.*InvalidBlocks*`) //bt.skipLoad(`.*TransitionTests*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { diff --git a/tests/init.go b/tests/init.go index d6b5b3043d..47bfac1859 100644 --- a/tests/init.go +++ b/tests/init.go @@ -28,15 +28,18 @@ import ( var Forks = map[string]*params.ChainConfig{ "Frontier": { ChainID: big.NewInt(1), + Bor: params.BorUnittestChainConfig.Bor, }, "Homestead": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "EIP150": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "EIP158": { ChainID: big.NewInt(1), @@ -44,6 +47,7 @@ var Forks = map[string]*params.ChainConfig{ EIP150Block: big.NewInt(0), EIP155Block: big.NewInt(0), EIP158Block: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "Byzantium": { ChainID: big.NewInt(1), @@ -53,6 +57,7 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), DAOForkBlock: big.NewInt(0), ByzantiumBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "Constantinople": { ChainID: big.NewInt(1), @@ -64,6 +69,7 @@ var Forks = map[string]*params.ChainConfig{ ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(10000000), + Bor: params.BorUnittestChainConfig.Bor, }, "ConstantinopleFix": { ChainID: big.NewInt(1), @@ -75,6 +81,7 @@ var Forks = map[string]*params.ChainConfig{ ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "Istanbul": { ChainID: big.NewInt(1), @@ -87,21 +94,25 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "FrontierToHomesteadAt5": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "HomesteadToEIP150At5": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), EIP150Block: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "HomesteadToDaoAt5": { ChainID: big.NewInt(1), HomesteadBlock: big.NewInt(0), DAOForkBlock: big.NewInt(5), DAOForkSupport: true, + Bor: params.BorUnittestChainConfig.Bor, }, "EIP158ToByzantiumAt5": { ChainID: big.NewInt(1), @@ -110,6 +121,7 @@ var Forks = map[string]*params.ChainConfig{ EIP155Block: big.NewInt(0), EIP158Block: big.NewInt(0), ByzantiumBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "ByzantiumToConstantinopleAt5": { ChainID: big.NewInt(1), @@ -119,6 +131,7 @@ var Forks = map[string]*params.ChainConfig{ EIP158Block: big.NewInt(0), ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "ByzantiumToConstantinopleFixAt5": { ChainID: big.NewInt(1), @@ -129,6 +142,7 @@ var Forks = map[string]*params.ChainConfig{ ByzantiumBlock: big.NewInt(0), ConstantinopleBlock: big.NewInt(5), PetersburgBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "ConstantinopleFixToIstanbulAt5": { ChainID: big.NewInt(1), @@ -140,6 +154,7 @@ var Forks = map[string]*params.ChainConfig{ ConstantinopleBlock: big.NewInt(0), PetersburgBlock: big.NewInt(0), IstanbulBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "Berlin": { ChainID: big.NewInt(1), @@ -153,6 +168,7 @@ var Forks = map[string]*params.ChainConfig{ IstanbulBlock: big.NewInt(0), MuirGlacierBlock: big.NewInt(0), BerlinBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "BerlinToLondonAt5": { ChainID: big.NewInt(1), @@ -167,6 +183,7 @@ var Forks = map[string]*params.ChainConfig{ MuirGlacierBlock: big.NewInt(0), BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(5), + Bor: params.BorUnittestChainConfig.Bor, }, "London": { ChainID: big.NewInt(1), @@ -181,6 +198,7 @@ var Forks = map[string]*params.ChainConfig{ MuirGlacierBlock: big.NewInt(0), BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, "ArrowGlacier": { ChainID: big.NewInt(1), @@ -196,6 +214,7 @@ var Forks = map[string]*params.ChainConfig{ BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(0), ArrowGlacierBlock: big.NewInt(0), + Bor: params.BorUnittestChainConfig.Bor, }, } From 2a677a5aff9ce9f01d051c46ea65458c1939b05e Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 23 Sep 2022 08:30:31 +0530 Subject: [PATCH 04/10] fix linters caused by previous merge (#526) --- ethclient/ethclient.go | 9 +++++++-- ethclient/gethclient/gethclient.go | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index c0b91b8569..13a2e46d25 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -533,18 +533,23 @@ func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" } + pending := big.NewInt(-1) + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(pending) == 0 { return "pending" } - finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { return "finalized" } - safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { return "safe" } + return hexutil.EncodeBig(number) } diff --git a/ethclient/gethclient/gethclient.go b/ethclient/gethclient/gethclient.go index ae147a40a3..2abeeaaf5c 100644 --- a/ethclient/gethclient/gethclient.go +++ b/ethclient/gethclient/gethclient.go @@ -183,18 +183,23 @@ func toBlockNumArg(number *big.Int) string { if number == nil { return "latest" } + pending := big.NewInt(-1) + finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(pending) == 0 { return "pending" } - finalized := big.NewInt(int64(rpc.FinalizedBlockNumber)) + if number.Cmp(finalized) == 0 { return "finalized" } - safe := big.NewInt(int64(rpc.SafeBlockNumber)) + if number.Cmp(safe) == 0 { return "safe" } + return hexutil.EncodeBig(number) } From ae5323de025ad40f4c452c013817b6c708912618 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 23 Sep 2022 11:42:10 +0530 Subject: [PATCH 05/10] new: unskip evm-benchmark tests --- core/state_transition.go | 20 +++++++++++--------- tests/block_test.go | 2 +- tests/state_test.go | 5 ++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index e867d032bf..3fc5a635e9 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -42,8 +42,10 @@ The state transitioning model does all the necessary work to work out a valid ne 3) Create a new state object if the recipient is \0*32 4) Value transfer == If contract creation == - 4a) Attempt to run transaction data - 4b) If valid, use result as code for the new state object + + 4a) Attempt to run transaction data + 4b) If valid, use result as code for the new state object + == end == 5) Run Script section 6) Derive new state root @@ -262,13 +264,13 @@ func (st *StateTransition) preCheck() error { // TransitionDb will transition the state by applying the current message and // returning the evm execution result with following fields. // -// - used gas: -// total gas used (including gas being refunded) -// - returndata: -// the returned data from evm -// - concrete execution error: -// various **EVM** error which aborts the execution, -// e.g. ErrOutOfGas, ErrExecutionReverted +// - used gas: +// total gas used (including gas being refunded) +// - returndata: +// the returned data from evm +// - concrete execution error: +// various **EVM** error which aborts the execution, +// e.g. ErrOutOfGas, ErrExecutionReverted // // However if any consensus issue encountered, return the error directly with // nil evm execution result. diff --git a/tests/block_test.go b/tests/block_test.go index a1d700d02e..6bdb98679f 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -53,7 +53,7 @@ func TestBlockchain(t *testing.T) { // FIXME POS-618 bt.skipLoad(`.*ValidBlocks*`) bt.skipLoad(`.*InvalidBlocks*`) - //bt.skipLoad(`.*TransitionTests*`) + bt.skipLoad(`.*TransitionTests*`) bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { if err := bt.checkFailure(t, test.Run(false)); err != nil { diff --git a/tests/state_test.go b/tests/state_test.go index eeb19570df..86f726336d 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -59,8 +59,8 @@ func TestState(t *testing.T) { st.skipLoad(`^stStaticCall/static_Call1MB`) // FIXME POS-618 - /*st.skipLoad(`.*micro/*`) - st.skipLoad(`.*main/*`) + //st.skipLoad(`.*micro/*`) + //st.skipLoad(`.*main/*`) st.skipLoad(`.*stSStoreTest*`) st.skipLoad(`.*stReturnDataTest*`) st.skipLoad(`.*stShift*`) @@ -115,7 +115,6 @@ func TestState(t *testing.T) { st.skipLoad(`.*stBadOpcode*`) st.skipLoad(`.*stStackTests*`) st.skipLoad(`.*stAttackTest*`) - */ // Broken tests: // Expected failures: From 2e7e99f1c5885690a89dbf9ba80ca85d0406e48e Mon Sep 17 00:00:00 2001 From: Shivam Sharma Date: Fri, 23 Sep 2022 12:43:44 +0530 Subject: [PATCH 06/10] chg : generalised state-sync integration test --- integration-tests/smoke_test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/integration-tests/smoke_test.sh b/integration-tests/smoke_test.sh index 6f41aa5c16..4181f9f20c 100644 --- a/integration-tests/smoke_test.sh +++ b/integration-tests/smoke_test.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +balanceInit=$(docker exec bor0 bash -c "bor attach /root/.bor/data/bor.ipc -exec 'Math.round(web3.fromWei(eth.getBalance(eth.accounts[0])))'") + delay=600 echo "Wait ${delay} seconds for state-sync..." @@ -16,7 +18,7 @@ fi echo "Found matic balance on account[0]: " $balance -if (( $balance <= 1001 )); then +if (( $balance <= $balanceInit )); then echo "Balance in bor network has not increased. This indicates that something is wrong with state sync." exit 1 fi @@ -30,4 +32,4 @@ else echo "Found checkpoint ID:" $checkpointID fi -echo "All tests have passed!" \ No newline at end of file +echo "All tests have passed!" From 77db80cc910ad1fa2a337513b577abe344033770 Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Fri, 23 Sep 2022 13:37:04 +0530 Subject: [PATCH 07/10] Added script to generate config.toml fromstart.sh (#518) * added go and bash script to get config out of start.sh and updated flagset.go * changed 'requiredblocks' flag back to 'eth.requiredblocks' * updated script * changed 'requiredblocks' flag back to 'eth.requiredblocks' * updated tests, and removed requiredblocks from json and hcl * addressed comments --- builder/files/config.toml | 2 +- docs/config.md | 2 + go.mod | 1 + go.sum | 2 + internal/cli/flagset/flagset.go | 9 + internal/cli/server/command_test.go | 2 +- internal/cli/server/config.go | 2 +- internal/cli/server/config_test.go | 3 - internal/cli/server/flags.go | 4 +- internal/cli/server/testdata/test.hcl | 4 - internal/cli/server/testdata/test.json | 3 - internal/cli/server/testdata/test.toml | 2 +- scripts/getconfig.go | 672 +++++++++++++++++++++++++ scripts/getconfig.sh | 87 ++++ 14 files changed, 779 insertions(+), 16 deletions(-) create mode 100644 scripts/getconfig.go create mode 100755 scripts/getconfig.sh diff --git a/builder/files/config.toml b/builder/files/config.toml index d8503e4351..84c3327bbf 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -12,7 +12,7 @@ syncmode = "full" # snapshot = true # ethstats = "" -# [requiredblocks] +# ["eth.requiredblocks"] [p2p] # maxpeers = 1 diff --git a/docs/config.md b/docs/config.md index 196a0bf388..aebd9e12b9 100644 --- a/docs/config.md +++ b/docs/config.md @@ -19,6 +19,8 @@ gcmode = "full" snapshot = true ethstats = "" +["eth.requiredblocks"] + [p2p] maxpeers = 30 maxpendpeers = 50 diff --git a/go.mod b/go.mod index 4e2073b42f..7a643a251c 100644 --- a/go.mod +++ b/go.mod @@ -122,6 +122,7 @@ require ( github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.0 // indirect github.com/opentracing/opentracing-go v1.1.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/posener/complete v1.1.1 // indirect diff --git a/go.sum b/go.sum index 49c8af9305..dc419821c6 100644 --- a/go.sum +++ b/go.sum @@ -401,6 +401,8 @@ github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mo github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 h1:oYW+YCJ1pachXTQmzR3rNLYGGz4g/UgFcjb28p/viDM= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= diff --git a/internal/cli/flagset/flagset.go b/internal/cli/flagset/flagset.go index d9e204f0e4..933fe59060 100644 --- a/internal/cli/flagset/flagset.go +++ b/internal/cli/flagset/flagset.go @@ -44,6 +44,15 @@ func (f *Flagset) Help() string { return str + strings.Join(items, "\n\n") } +func (f *Flagset) GetAllFlags() []string { + flags := []string{} + for _, flag := range f.flags { + flags = append(flags, flag.Name) + } + + return flags +} + // MarkDown implements cli.MarkDown interface func (f *Flagset) MarkDown() string { if len(f.flags) == 0 { diff --git a/internal/cli/server/command_test.go b/internal/cli/server/command_test.go index 9006559d0f..ab28de5ee6 100644 --- a/internal/cli/server/command_test.go +++ b/internal/cli/server/command_test.go @@ -24,7 +24,7 @@ func TestFlags(t *testing.T) { "--dev.period", "2", "--datadir", "./data", "--maxpeers", "30", - "--requiredblocks", "a=b", + "--eth.requiredblocks", "a=b", "--http.api", "eth,web3,bor", } err := c.extractFlags(args) diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index c6193e6e76..ec44ade8d2 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -45,7 +45,7 @@ type Config struct { Identity string `hcl:"identity,optional" toml:"identity,optional"` // RequiredBlocks is a list of required (block number, hash) pairs to accept - RequiredBlocks map[string]string `hcl:"requiredblocks,optional" toml:"requiredblocks,optional"` + RequiredBlocks map[string]string `hcl:"eth.requiredblocks,optional" toml:"eth.requiredblocks,optional"` // LogLevel is the level of the logs to put out LogLevel string `hcl:"log-level,optional" toml:"log-level,optional"` diff --git a/internal/cli/server/config_test.go b/internal/cli/server/config_test.go index bf7787ac2e..5f3118996b 100644 --- a/internal/cli/server/config_test.go +++ b/internal/cli/server/config_test.go @@ -108,9 +108,6 @@ func TestConfigLoadFile(t *testing.T) { assert.Equal(t, config, &Config{ DataDir: "./data", - RequiredBlocks: map[string]string{ - "a": "b", - }, P2P: &P2PConfig{ MaxPeers: 30, }, diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 327b0df338..82af795af6 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -56,8 +56,8 @@ func (c *Command) Flags() *flagset.Flagset { Default: c.cliConfig.GcMode, }) f.MapStringFlag(&flagset.MapStringFlag{ - Name: "requiredblocks", - Usage: "Comma separated block number-to-hash mappings to enforce (=)", + Name: "eth.requiredblocks", + Usage: "Comma separated block number-to-hash mappings to require for peering (=)", Value: &c.cliConfig.RequiredBlocks, }) f.BoolFlag(&flagset.BoolFlag{ diff --git a/internal/cli/server/testdata/test.hcl b/internal/cli/server/testdata/test.hcl index 208fdc51c1..44138970fc 100644 --- a/internal/cli/server/testdata/test.hcl +++ b/internal/cli/server/testdata/test.hcl @@ -1,9 +1,5 @@ datadir = "./data" -requiredblocks = { - a = "b" -} - p2p { maxpeers = 30 } diff --git a/internal/cli/server/testdata/test.json b/internal/cli/server/testdata/test.json index 467835e755..a08e5aceb1 100644 --- a/internal/cli/server/testdata/test.json +++ b/internal/cli/server/testdata/test.json @@ -1,8 +1,5 @@ { "datadir": "./data", - "requiredblocks": { - "a": "b" - }, "p2p": { "maxpeers": 30 }, diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index 2277e03664..6d07abdad7 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -1,6 +1,6 @@ datadir = "./data" -[requiredblocks] +["eth.requiredblocks"] a = "b" [p2p] diff --git a/scripts/getconfig.go b/scripts/getconfig.go new file mode 100644 index 0000000000..9b5e6bf2b3 --- /dev/null +++ b/scripts/getconfig.go @@ -0,0 +1,672 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "os" + "strconv" + "strings" + + "github.com/pelletier/go-toml" + + "github.com/ethereum/go-ethereum/internal/cli/server" +) + +// YesFV: Both, Flags and their values has changed +// YesF: Only the Flag has changed, not their value +var flagMap = map[string][]string{ + "networkid": {"notABoolFlag", "YesFV"}, + "miner.gastarget": {"notABoolFlag", "No"}, + "pprof": {"BoolFlag", "No"}, + "pprof.port": {"notABoolFlag", "No"}, + "pprof.addr": {"notABoolFlag", "No"}, + "pprof.memprofilerate": {"notABoolFlag", "No"}, + "pprof.blockprofilerate": {"notABoolFlag", "No"}, + "pprof.cpuprofile": {"notABoolFlag", "No"}, + "jsonrpc.corsdomain": {"notABoolFlag", "YesF"}, + "jsonrpc.vhosts": {"notABoolFlag", "YesF"}, + "http.modules": {"notABoolFlag", "YesF"}, + "ws.modules": {"notABoolFlag", "YesF"}, + "config": {"notABoolFlag", "No"}, + "datadir.ancient": {"notABoolFlag", "No"}, + "datadir.minfreedisk": {"notABoolFlag", "No"}, + "usb": {"BoolFlag", "No"}, + "pcscdpath": {"notABoolFlag", "No"}, + "mainnet": {"BoolFlag", "No"}, + "goerli": {"BoolFlag", "No"}, + "bor-mumbai": {"BoolFlag", "No"}, + "bor-mainnet": {"BoolFlag", "No"}, + "rinkeby": {"BoolFlag", "No"}, + "ropsten": {"BoolFlag", "No"}, + "sepolia": {"BoolFlag", "No"}, + "kiln": {"BoolFlag", "No"}, + "exitwhensynced": {"BoolFlag", "No"}, + "light.serve": {"notABoolFlag", "No"}, + "light.ingress": {"notABoolFlag", "No"}, + "light.egress": {"notABoolFlag", "No"}, + "light.maxpeers": {"notABoolFlag", "No"}, + "ulc.servers": {"notABoolFlag", "No"}, + "ulc.fraction": {"notABoolFlag", "No"}, + "ulc.onlyannounce": {"BoolFlag", "No"}, + "light.nopruning": {"BoolFlag", "No"}, + "light.nosyncserve": {"BoolFlag", "No"}, + "dev.gaslimit": {"notABoolFlag", "No"}, + "ethash.cachedir": {"notABoolFlag", "No"}, + "ethash.cachesinmem": {"notABoolFlag", "No"}, + "ethash.cachesondisk": {"notABoolFlag", "No"}, + "ethash.cacheslockmmap": {"BoolFlag", "No"}, + "ethash.dagdir": {"notABoolFlag", "No"}, + "ethash.dagsinmem": {"notABoolFlag", "No"}, + "ethash.dagsondisk": {"notABoolFlag", "No"}, + "ethash.dagslockmmap": {"BoolFlag", "No"}, + "fdlimit": {"notABoolFlag", "No"}, + "signer": {"notABoolFlag", "No"}, + "authrpc.jwtsecret": {"notABoolFlag", "No"}, + "authrpc.addr": {"notABoolFlag", "No"}, + "authrpc.port": {"notABoolFlag", "No"}, + "authrpc.vhosts": {"notABoolFlag", "No"}, + "graphql.corsdomain": {"notABoolFlag", "No"}, + "graphql.vhosts": {"notABoolFlag", "No"}, + "rpc.evmtimeout": {"notABoolFlag", "No"}, + "rpc.allow-unprotected-txs": {"BoolFlag", "No"}, + "jspath": {"notABoolFlag", "No"}, + "exec": {"notABoolFlag", "No"}, + "preload": {"notABoolFlag", "No"}, + "discovery.dns": {"notABoolFlag", "No"}, + "netrestrict": {"notABoolFlag", "No"}, + "nodekey": {"notABoolFlag", "No"}, + "nodekeyhex": {"notABoolFlag", "No"}, + "miner.threads": {"notABoolFlag", "No"}, + "miner.notify": {"notABoolFlag", "No"}, + "miner.notify.full": {"BoolFlag", "No"}, + "miner.recommit": {"notABoolFlag", "No"}, + "miner.noverify": {"BoolFlag", "No"}, + "vmdebug": {"BoolFlag", "No"}, + "fakepow": {"BoolFlag", "No"}, + "nocompaction": {"BoolFlag", "No"}, + "metrics.addr": {"notABoolFlag", "No"}, + "metrics.port": {"notABoolFlag", "No"}, + "whitelist": {"notABoolFlag", "No"}, + "snapshot": {"BoolFlag", "YesF"}, + "bloomfilter.size": {"notABoolFlag", "No"}, + "bor.logs": {"BoolFlag", "No"}, + "override.arrowglacier": {"notABoolFlag", "No"}, + "override.terminaltotaldifficulty": {"notABoolFlag", "No"}, + "verbosity": {"notABoolFlag", "YesFV"}, + "ws.origins": {"notABoolFlag", "YesF"}, +} + +// map from cli flags to corresponding toml tags +var nameTagMap = map[string]string{ + "chain": "chain", + "identity": "identity", + "log-level": "log-level", + "datadir": "datadir", + "keystore": "keystore", + "syncmode": "syncmode", + "gcmode": "gcmode", + "eth.requiredblocks": "eth.requiredblocks", + "0-snapshot": "snapshot", + "url": "bor.heimdall", + "bor.without": "bor.withoutheimdall", + "grpc-address": "bor.heimdallgRPC", + "locals": "txpool.locals", + "nolocals": "txpool.nolocals", + "journal": "txpool.journal", + "rejournal": "txpool.rejournal", + "pricelimit": "txpool.pricelimit", + "pricebump": "txpool.pricebump", + "accountslots": "txpool.accountslots", + "globalslots": "txpool.globalslots", + "accountqueue": "txpool.accountqueue", + "globalqueue": "txpool.globalqueue", + "lifetime": "txpool.lifetime", + "mine": "mine", + "etherbase": "miner.etherbase", + "extradata": "miner.extradata", + "gaslimit": "miner.gaslimit", + "gasprice": "miner.gasprice", + "ethstats": "ethstats", + "blocks": "gpo.blocks", + "percentile": "gpo.percentile", + "maxprice": "gpo.maxprice", + "ignoreprice": "gpo.ignoreprice", + "cache": "cache", + "1-database": "cache.database", + "trie": "cache.trie", + "trie.journal": "cache.journal", + "trie.rejournal": "cache.rejournal", + "gc": "cache.gc", + "1-snapshot": "cache.snapshot", + "noprefetch": "cache.noprefetch", + "preimages": "cache.preimages", + "txlookuplimit": "txlookuplimit", + "gascap": "rpc.gascap", + "txfeecap": "rpc.txfeecap", + "ipcdisable": "ipcdisable", + "ipcpath": "ipcpath", + "1-corsdomain": "http.corsdomain", + "1-vhosts": "http.vhosts", + "2-corsdomain": "ws.corsdomain", + "2-vhosts": "ws.vhosts", + "3-corsdomain": "graphql.corsdomain", + "3-vhosts": "graphql.vhosts", + "1-enabled": "http", + "1-host": "http.addr", + "1-port": "http.port", + "1-prefix": "http.rpcprefix", + "1-api": "http.api", + "2-enabled": "ws", + "2-host": "ws.addr", + "2-port": "ws.port", + "2-prefix": "ws.rpcprefix", + "2-api": "ws.api", + "3-enabled": "graphql", + "bind": "bind", + "0-port": "port", + "bootnodes": "bootnodes", + "maxpeers": "maxpeers", + "maxpendpeers": "maxpendpeers", + "nat": "nat", + "nodiscover": "nodiscover", + "v5disc": "v5disc", + "metrics": "metrics", + "expensive": "metrics.expensive", + "influxdb": "metrics.influxdb", + "endpoint": "metrics.influxdb.endpoint", + "0-database": "metrics.influxdb.database", + "username": "metrics.influxdb.username", + "0-password": "metrics.influxdb.password", + "tags": "metrics.influxdb.tags", + "prometheus-addr": "metrics.prometheus-addr", + "opencollector-endpoint": "metrics.opencollector-endpoint", + "influxdbv2": "metrics.influxdbv2", + "token": "metrics.influxdb.token", + "bucket": "metrics.influxdb.bucket", + "organization": "metrics.influxdb.organization", + "unlock": "unlock", + "1-password": "password", + "allow-insecure-unlock": "allow-insecure-unlock", + "lightkdf": "lightkdf", + "disable-bor-wallet": "disable-bor-wallet", + "addr": "grpc.addr", + "dev": "dev", + "period": "dev.period", +} + +var removedFlagsAndValues = map[string]string{} + +var replacedFlagsMapFlagAndValue = map[string]map[string]map[string]string{ + "networkid": { + "flag": { + "networkid": "chain", + }, + "value": { + "'137'": "mainnet", + "137": "mainnet", + "'80001'": "mumbai", + "80001": "mumbai", + }, + }, + "verbosity": { + "flag": { + "verbosity": "log-level", + }, + "value": { + "0": "SILENT", + "1": "ERROR", + "2": "WARN", + "3": "INFO", + "4": "DEBUG", + "5": "DETAIL", + }, + }, +} + +var replacedFlagsMapFlag = map[string]string{ + "ws.origins": "ws.corsdomain", +} + +var currentBoolFlags = []string{ + "snapshot", + "bor.withoutheimdall", + "txpool.nolocals", + "mine", + "cache.noprefetch", + "cache.preimages", + "ipcdisable", + "http", + "ws", + "graphql", + "nodiscover", + "v5disc", + "metrics", + "metrics.expensive", + "metrics.influxdb", + "metrics.influxdbv2", + "allow-insecure-unlock", + "lightkdf", + "disable-bor-wallet", + "dev", +} + +func contains(s []string, str string) (bool, int) { + for ind, v := range s { + if v == str || v == "-"+str || v == "--"+str { + return true, ind + } + } + + return false, -1 +} + +func indexOf(s []string, str string) int { + for k, v := range s { + if v == str || v == "-"+str || v == "--"+str { + return k + } + } + + return -1 +} + +func remove1(s []string, idx int) []string { + removedFlagsAndValues[s[idx]] = "" + return append(s[:idx], s[idx+1:]...) +} + +func remove2(s []string, idx int) []string { + removedFlagsAndValues[s[idx]] = s[idx+1] + return append(s[:idx], s[idx+2:]...) +} + +func checkFlag(allFlags []string, checkFlags []string) []string { + outOfDateFlags := []string{} + + for _, flag := range checkFlags { + t1, _ := contains(allFlags, flag) + if !t1 { + outOfDateFlags = append(outOfDateFlags, flag) + } + } + + return outOfDateFlags +} + +func checkFileExists(path string) bool { + _, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + fmt.Println("WARN: File does not exist", path) + return false + } else { + return true + } +} + +func writeTempStaticJSON(path string) { + data, err := os.ReadFile(path) + if err != nil { + log.Fatal(err) + } + + var conf interface{} + if err := json.Unmarshal(data, &conf); err != nil { + log.Fatal(err) + } + + temparr := []string{} + for _, item := range conf.([]interface{}) { + temparr = append(temparr, item.(string)) + } + + // write to a temp file + err = os.WriteFile("./tempStaticNodes.json", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } +} + +func writeTempStaticTrustedTOML(path string) { + data, err := toml.LoadFile(path) + if err != nil { + log.Fatal(err) + } + + if data.Has("Node.P2P.StaticNodes") { + temparr := []string{} + for _, item := range data.Get("Node.P2P.StaticNodes").([]interface{}) { + temparr = append(temparr, item.(string)) + } + + err = os.WriteFile("./tempStaticNodes.toml", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } + } + + if data.Has("Node.P2P.TrustedNodes") { + temparr := []string{} + for _, item := range data.Get("Node.P2P.TrustedNodes").([]interface{}) { + temparr = append(temparr, item.(string)) + } + + err = os.WriteFile("./tempTrustedNodes.toml", []byte(strings.Join(temparr, "\", \"")), 0600) + if err != nil { + log.Fatal(err) + } + } +} + +func getStaticTrustedNodes(args []string) { + // if config flag is passed, it should be only a .toml file + t1, t2 := contains(args, "config") + // nolint: nestif + if t1 { + path := args[t2+1] + if !checkFileExists(path) { + return + } + + if path[len(path)-4:] == "toml" { + writeTempStaticTrustedTOML(path) + } else { + fmt.Println("only TOML config file is supported through CLI") + } + } else { + path := "~/.bor/data/bor/static-nodes.json" + if !checkFileExists(path) { + return + } + writeTempStaticJSON(path) + } +} + +func getFlagsToCheck(args []string) []string { + flagsToCheck := []string{} + + for _, item := range args { + if strings.HasPrefix(item, "-") { + if item[1] == '-' { + temp := item[2:] + flagsToCheck = append(flagsToCheck, temp) + } else { + temp := item[1:] + flagsToCheck = append(flagsToCheck, temp) + } + } + } + + return flagsToCheck +} + +func getFlagType(flag string) string { + return flagMap[flag][0] +} + +func updateArgsClean(args []string, outOfDateFlags []string) []string { + updatedArgs := []string{} + updatedArgs = append(updatedArgs, args...) + + // iterate through outOfDateFlags and remove the flags from updatedArgs along with their value (if any) + for _, item := range outOfDateFlags { + idx := indexOf(updatedArgs, item) + + if getFlagType(item) == "BoolFlag" { + // remove the element at index idx + updatedArgs = remove1(updatedArgs, idx) + } else { + // remove the element at index idx and idx + 1 + updatedArgs = remove2(updatedArgs, idx) + } + } + + return updatedArgs +} + +func updateArgsAdd(args []string) []string { + for flag, value := range removedFlagsAndValues { + if strings.HasPrefix(flag, "--") { + flag = flag[2:] + } else { + flag = flag[1:] + } + + if flagMap[flag][1] == "YesFV" { + temp := "--" + replacedFlagsMapFlagAndValue[flag]["flag"][flag] + " " + replacedFlagsMapFlagAndValue[flag]["value"][value] + args = append(args, temp) + } else if flagMap[flag][1] == "YesF" { + temp := "--" + replacedFlagsMapFlag[flag] + " " + value + args = append(args, temp) + } + } + + return args +} + +func handleGRPC(args []string, updatedArgs []string) []string { + var newUpdatedArgs []string + + var addr string + + var port string + + newUpdatedArgs = append(newUpdatedArgs, updatedArgs...) + + for i, val := range args { + if strings.Contains(val, "pprof.addr") && strings.Contains(val, "-") { + addr = args[i+1] + } + + if strings.Contains(val, "pprof.port") && strings.Contains(val, "-") { + port = args[i+1] + } + } + + newUpdatedArgs = append(newUpdatedArgs, "--grpc.addr") + newUpdatedArgs = append(newUpdatedArgs, addr+":"+port) + + return newUpdatedArgs +} + +func dumpFlags(args []string) { + err := os.WriteFile("./temp", []byte(strings.Join(args, " ")), 0600) + if err != nil { + fmt.Println("Error in WriteFile") + } else { + fmt.Println("WriteFile Done") + } +} + +// nolint: gocognit +func commentFlags(path string, updatedArgs []string) { + const cache = "[cache]" + + const telemetry = "[telemetry]" + + // snapshot: "[cache]" + cacheFlag := 0 + + // corsdomain, vhosts, enabled, host, api, port, prefix: "[p2p]", " [jsonrpc.http]", " [jsonrpc.ws]", " [jsonrpc.graphql]" + p2pHttpWsGraphFlag := -1 + + // database: "[telemetry]", "[cache]" + databaseFlag := -1 + + // password: "[telemetry]", "[accounts]" + passwordFlag := -1 + + ignoreLineFlag := false + + input, err := os.ReadFile(path) + if err != nil { + log.Fatalln(err) + } + + lines := strings.Split(string(input), "\n") + + var newLines []string + newLines = append(newLines, lines...) + + for i, line := range lines { + if line == cache { + cacheFlag += 1 + } + + if line == "[p2p]" || line == " [jsonrpc.http]" || line == " [jsonrpc.ws]" || line == " [jsonrpc.graphql]" { + p2pHttpWsGraphFlag += 1 + } + + if line == telemetry || line == cache { + databaseFlag += 1 + } + + if line == telemetry || line == "[accounts]" { + passwordFlag += 1 + } + + if line == "[\"eth.requiredblocks\"]" || line == " [telemetry.influx.tags]" { + ignoreLineFlag = true + } else if line != "" { + if strings.HasPrefix(strings.Fields(line)[0], "[") { + ignoreLineFlag = false + } + } + + // nolint: nestif + if !(strings.HasPrefix(line, "[") || strings.HasPrefix(line, " [") || strings.HasPrefix(line, " [") || line == "" || ignoreLineFlag) { + flag := strings.Fields(line)[0] + if flag == "snapshot" { + flag = strconv.Itoa(cacheFlag) + "-" + flag + } else if flag == "corsdomain" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "vhosts" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "enabled" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "host" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "api" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "port" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "prefix" { + flag = strconv.Itoa(p2pHttpWsGraphFlag) + "-" + flag + } else if flag == "database" { + flag = strconv.Itoa(databaseFlag) + "-" + flag + } else if flag == "password" { + flag = strconv.Itoa(passwordFlag) + "-" + flag + } + + if flag != "static-nodes" && flag != "trusted-nodes" { + flag = nameTagMap[flag] + + tempFlag := false + + for _, val := range updatedArgs { + if strings.Contains(val, flag) && (strings.Contains(val, "-") || strings.Contains(val, "--")) { + tempFlag = true + } + } + + if !tempFlag || flag == "" { + newLines[i] = "# " + line + } + } + } + } + + output := strings.Join(newLines, "\n") + + err = os.WriteFile(path, []byte(output), 0600) + if err != nil { + log.Fatalln(err) + } +} + +func checkBoolFlags(val string) bool { + returnFlag := false + + if strings.Contains(val, "=") { + val = strings.Split(val, "=")[0] + } + + for _, flag := range currentBoolFlags { + if val == "-"+flag || val == "--"+flag { + returnFlag = true + } + } + + return returnFlag +} + +func beautifyArgs(args []string) ([]string, []string) { + newArgs := []string{} + + ignoreForNow := []string{} + + temp := []string{} + + for _, val := range args { + // nolint: nestif + if !(checkBoolFlags(val)) { + if strings.HasPrefix(val, "-") { + if strings.Contains(val, "=") { + temparr := strings.Split(val, "=") + newArgs = append(newArgs, temparr...) + } else { + newArgs = append(newArgs, val) + } + } else { + newArgs = append(newArgs, val) + } + } else { + ignoreForNow = append(ignoreForNow, val) + } + } + + for j, val := range newArgs { + if val == "-unlock" || val == "--unlock" { + temp = append(temp, "--miner.etherbase") + temp = append(temp, newArgs[j+1]) + } + } + + newArgs = append(newArgs, temp...) + + return newArgs, ignoreForNow +} + +func main() { + const notYet = "notYet" + + temp := os.Args[1] + args := os.Args[2:] + + args, ignoreForNow := beautifyArgs(args) + + c := server.Command{} + flags := c.Flags() + allFlags := flags.GetAllFlags() + flagsToCheck := getFlagsToCheck(args) + + if temp == notYet { + getStaticTrustedNodes(args) + } + + outOfDateFlags := checkFlag(allFlags, flagsToCheck) + updatedArgs := updateArgsClean(args, outOfDateFlags) + updatedArgs = updateArgsAdd(updatedArgs) + updatedArgs = handleGRPC(args, updatedArgs) + + if temp == notYet { + updatedArgs = append(updatedArgs, ignoreForNow...) + dumpFlags(updatedArgs) + } + + if temp != notYet { + updatedArgs = append(updatedArgs, ignoreForNow...) + commentFlags(temp, updatedArgs) + } +} diff --git a/scripts/getconfig.sh b/scripts/getconfig.sh new file mode 100755 index 0000000000..26d5d0138c --- /dev/null +++ b/scripts/getconfig.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env sh + +# Instructions: +# Execute `./getconfig.sh`, and follow the instructions displayed on the terminal +# The `*-config.toml` file will be created in the same directory as start.sh +# It is recommended to check the flags generated in config.toml + + +read -p "* Path to start.sh: " startPath +# check if start.sh is present +if [[ ! -f $startPath ]] +then + echo "Error: start.sh do not exist." + exit 1 +fi +read -p "* Your validator address (e.g. 0xca67a8D767e45056DC92384b488E9Af654d78DE2), or press Enter to skip if running a sentry node: " ADD + +echo "\nThank you, your inputs are:" +echo "Path to start.sh: "$startPath +echo "Address: "$ADD + +confPath=${startPath%.sh}"-config.toml" +echo "Path to the config file: "$confPath +# check if config.toml is present +if [[ -f $confPath ]] +then + echo "WARN: config.toml exists, data will be overwritten." +fi + +tmpDir="$(mktemp -d -t ./temp-dir-XXXXXXXXXXX || oops "Can't create temporary directory")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +# SHA1 hash of `tempStart` -> `3305fe263dd4a999d58f96deb064e21bb70123d9` +sed 's/bor --/go run getconfig.go notYet --/g' $startPath > $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +chmod +x $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +$tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $ADD +rm $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh + + +sed -i '' "s%*%'*'%g" ./temp + +# read the flags from `./temp` +dumpconfigflags=$(head -1 ./temp) + +# run the dumpconfig command with the flags from `./temp` +command="bor dumpconfig "$dumpconfigflags" > "$confPath +bash -c "$command" + +rm ./temp + +if [[ -f ./tempStaticNodes.json ]] +then + echo "JSON StaticNodes found" + staticnodesjson=$(head -1 ./tempStaticNodes.json) + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodesjson}\"\]%" $confPath + rm ./tempStaticNodes.json +elif [[ -f ./tempStaticNodes.toml ]] +then + echo "TOML StaticNodes found" + staticnodestoml=$(head -1 ./tempStaticNodes.toml) + sed -i '' "s%static-nodes = \[\]%static-nodes = \[\"${staticnodestoml}\"\]%" $confPath + rm ./tempStaticNodes.toml +else + echo "neither JSON nor TOML StaticNodes found" +fi + +if [[ -f ./tempTrustedNodes.toml ]] +then + echo "TOML TrustedNodes found" + trustednodestoml=$(head -1 ./tempTrustedNodes.toml) + sed -i '' "s%trusted-nodes = \[\]%trusted-nodes = \[\"${trustednodestoml}\"\]%" $confPath + rm ./tempTrustedNodes.toml +else + echo "neither JSON nor TOML TrustedNodes found" +fi + +# comment flags in $configPath that were not passed through $startPath +# SHA1 hash of `tempStart` -> `3305fe263dd4a999d58f96deb064e21bb70123d9` +sed "s%bor --%go run getconfig.go ${confPath} --%" $startPath > $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +chmod +x $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh +$tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh $ADD +rm $tmpDir/3305fe263dd4a999d58f96deb064e21bb70123d9.sh + +exit 0 From 0eec63716b1d65eefa97cfeb4dbb147e8d8ceb84 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Fri, 23 Sep 2022 16:23:32 +0530 Subject: [PATCH 08/10] new: unskip stCreateTest --- tests/state_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/state_test.go b/tests/state_test.go index 86f726336d..a39f126fbe 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -59,8 +59,8 @@ func TestState(t *testing.T) { st.skipLoad(`^stStaticCall/static_Call1MB`) // FIXME POS-618 - //st.skipLoad(`.*micro/*`) - //st.skipLoad(`.*main/*`) + st.skipLoad(`.*micro/*`) + st.skipLoad(`.*main/*`) st.skipLoad(`.*stSStoreTest*`) st.skipLoad(`.*stReturnDataTest*`) st.skipLoad(`.*stShift*`) @@ -97,7 +97,7 @@ func TestState(t *testing.T) { st.skipLoad(`.*stChangedEIP150*`) st.skipLoad(`.*stLogTests*`) st.skipLoad(`.*stSLoadTest*`) - st.skipLoad(`.*stCreateTest*`) + //st.skipLoad(`.*stCreateTest*`) st.skipLoad(`.*stDelegatecallTestHomestead*`) st.skipLoad(`.*stCallDelegateCodesHomestead*`) st.skipLoad(`.*VMTests*`) From 2db0dc860655db046603354112a7184e9a5d0117 Mon Sep 17 00:00:00 2001 From: Raneet Debnath Date: Mon, 26 Sep 2022 17:26:05 +0530 Subject: [PATCH 09/10] new: comments and cleanup --- tests/block_test.go | 2 +- tests/state_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 6bdb98679f..e2aa1efea7 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -50,7 +50,7 @@ func TestBlockchain(t *testing.T) { // using 4.6 TGas bt.skipLoad(`.*randomStatetest94.json.*`) - // FIXME POS-618 + // See POS-618 bt.skipLoad(`.*ValidBlocks*`) bt.skipLoad(`.*InvalidBlocks*`) bt.skipLoad(`.*TransitionTests*`) diff --git a/tests/state_test.go b/tests/state_test.go index a39f126fbe..f18b84d16e 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -58,7 +58,7 @@ func TestState(t *testing.T) { // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) - // FIXME POS-618 + // See POS-618 st.skipLoad(`.*micro/*`) st.skipLoad(`.*main/*`) st.skipLoad(`.*stSStoreTest*`) @@ -97,7 +97,6 @@ func TestState(t *testing.T) { st.skipLoad(`.*stChangedEIP150*`) st.skipLoad(`.*stLogTests*`) st.skipLoad(`.*stSLoadTest*`) - //st.skipLoad(`.*stCreateTest*`) st.skipLoad(`.*stDelegatecallTestHomestead*`) st.skipLoad(`.*stCallDelegateCodesHomestead*`) st.skipLoad(`.*VMTests*`) From 457314783f66d54b929fc487f5074616262704ca Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Tue, 27 Sep 2022 13:36:42 +0530 Subject: [PATCH 10/10] internal/cli/server: fix flag behaviour (#529) * remove setting maxpeers to 0 for nodiscover flag * set default prometheus and open-collector endpoint * skip building grpc address from pprof address and port * fix: linters * fix and improve tests * use loopback address for prometheus and open-collector endpoint * add logs for prometheus and open-collector setup * updated the script to handle prometheus-addr * updated builder/files/config.toml Co-authored-by: Pratik Patil --- builder/files/config.toml | 4 +- internal/cli/server/config.go | 7 +- internal/cli/server/config_legacy_test.go | 151 +++------------------- internal/cli/server/server.go | 4 + internal/cli/server/testdata/test.toml | 6 +- scripts/getconfig.go | 35 +++-- 6 files changed, 54 insertions(+), 153 deletions(-) diff --git a/builder/files/config.toml b/builder/files/config.toml index 84c3327bbf..ce9fd782d0 100644 --- a/builder/files/config.toml +++ b/builder/files/config.toml @@ -96,8 +96,8 @@ syncmode = "full" [telemetry] metrics = true # expensive = false - prometheus-addr = "127.0.0.1:7071" - # opencollector-endpoint = "" + # prometheus-addr = "127.0.0.1:7071" + # opencollector-endpoint = "127.0.0.1:4317" # [telemetry.influx] # influxdb = false # endpoint = "" diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index ec44ade8d2..b25b620dd8 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -479,8 +479,8 @@ func DefaultConfig() *Config { Telemetry: &TelemetryConfig{ Enabled: false, Expensive: false, - PrometheusAddr: "", - OpenCollectorEndpoint: "", + PrometheusAddr: "127.0.0.1:7071", + OpenCollectorEndpoint: "127.0.0.1:4317", InfluxDB: &InfluxDBConfig{ V1Enabled: false, Endpoint: "", @@ -996,8 +996,7 @@ func (c *Config) buildNode() (*node.Config, error) { } if c.P2P.NoDiscover { - // Disable networking, for now, we will not even allow incomming connections - cfg.P2P.MaxPeers = 0 + // Disable peer discovery cfg.P2P.NoDiscovery = true } diff --git a/internal/cli/server/config_legacy_test.go b/internal/cli/server/config_legacy_test.go index e22639fa7e..29cefdd7bf 100644 --- a/internal/cli/server/config_legacy_test.go +++ b/internal/cli/server/config_legacy_test.go @@ -6,9 +6,6 @@ import ( "time" "github.com/stretchr/testify/assert" - - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/params" ) func TestConfigLegacy(t *testing.T) { @@ -17,139 +14,23 @@ func TestConfigLegacy(t *testing.T) { expectedConfig, err := readLegacyConfig(path) assert.NoError(t, err) - testConfig := &Config{ - Chain: "mainnet", - Identity: Hostname(), - RequiredBlocks: map[string]string{ - "a": "b", - }, - LogLevel: "INFO", - DataDir: "./data", - P2P: &P2PConfig{ - MaxPeers: 30, - MaxPendPeers: 50, - Bind: "0.0.0.0", - Port: 30303, - NoDiscover: false, - NAT: "any", - Discovery: &P2PDiscovery{ - V5Enabled: false, - Bootnodes: []string{}, - BootnodesV4: []string{}, - BootnodesV5: []string{}, - StaticNodes: []string{}, - TrustedNodes: []string{}, - DNS: []string{}, - }, - }, - Heimdall: &HeimdallConfig{ - URL: "http://localhost:1317", - Without: false, - }, - SyncMode: "full", - GcMode: "full", - Snapshot: true, - TxPool: &TxPoolConfig{ - Locals: []string{}, - NoLocals: false, - Journal: "transactions.rlp", - Rejournal: 1 * time.Hour, - PriceLimit: 1, - PriceBump: 10, - AccountSlots: 16, - GlobalSlots: 32768, - AccountQueue: 16, - GlobalQueue: 32768, - LifeTime: 1 * time.Second, - }, - Sealer: &SealerConfig{ - Enabled: false, - Etherbase: "", - GasCeil: 30000000, - GasPrice: big.NewInt(1 * params.GWei), - ExtraData: "", - }, - Gpo: &GpoConfig{ - Blocks: 20, - Percentile: 60, - MaxPrice: big.NewInt(5000 * params.GWei), - IgnorePrice: big.NewInt(4), - }, - JsonRPC: &JsonRPCConfig{ - IPCDisable: false, - IPCPath: "", - GasCap: ethconfig.Defaults.RPCGasCap, - TxFeeCap: ethconfig.Defaults.RPCTxFeeCap, - Http: &APIConfig{ - Enabled: false, - Port: 8545, - Prefix: "", - Host: "localhost", - API: []string{"eth", "net", "web3", "txpool", "bor"}, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - Ws: &APIConfig{ - Enabled: false, - Port: 8546, - Prefix: "", - Host: "localhost", - API: []string{"net", "web3"}, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - Graphql: &APIConfig{ - Enabled: false, - Cors: []string{"localhost"}, - VHost: []string{"localhost"}, - }, - }, - Ethstats: "", - Telemetry: &TelemetryConfig{ - Enabled: false, - Expensive: false, - PrometheusAddr: "", - OpenCollectorEndpoint: "", - InfluxDB: &InfluxDBConfig{ - V1Enabled: false, - Endpoint: "", - Database: "", - Username: "", - Password: "", - Tags: map[string]string{}, - V2Enabled: false, - Token: "", - Bucket: "", - Organization: "", - }, - }, - Cache: &CacheConfig{ - Cache: 1024, - PercDatabase: 50, - PercTrie: 15, - PercGc: 25, - PercSnapshot: 10, - Journal: "triecache", - Rejournal: 1 * time.Second, - NoPrefetch: false, - Preimages: false, - TxLookupLimit: 2350000, - }, - Accounts: &AccountsConfig{ - Unlock: []string{}, - PasswordFile: "", - AllowInsecureUnlock: false, - UseLightweightKDF: false, - DisableBorWallet: true, - }, - GRPC: &GRPCConfig{ - Addr: ":3131", - }, - Developer: &DeveloperConfig{ - Enabled: false, - Period: 0, - }, + testConfig := DefaultConfig() + + testConfig.DataDir = "./data" + testConfig.Snapshot = false + testConfig.RequiredBlocks = map[string]string{ + "31000000": "0x2087b9e2b353209c2c21e370c82daa12278efd0fe5f0febe6c29035352cf050e", + "32000000": "0x875500011e5eecc0c554f95d07b31cf59df4ca2505f4dbbfffa7d4e4da917c68", } + testConfig.P2P.MaxPeers = 30 + testConfig.TxPool.Locals = []string{} + testConfig.TxPool.LifeTime = time.Second + testConfig.Sealer.Enabled = true + testConfig.Sealer.GasCeil = 30000000 + testConfig.Sealer.GasPrice = big.NewInt(1000000000) + testConfig.Gpo.IgnorePrice = big.NewInt(4) + testConfig.Cache.Cache = 1024 + testConfig.Cache.Rejournal = time.Second assert.Equal(t, expectedConfig, testConfig) } diff --git a/internal/cli/server/server.go b/internal/cli/server/server.go index 736b0d3bb7..905de36e09 100644 --- a/internal/cli/server/server.go +++ b/internal/cli/server/server.go @@ -305,6 +305,8 @@ func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error } }() + log.Info("Enabling metrics export to prometheus", "path", fmt.Sprintf("http://%s/debug/metrics/prometheus", config.PrometheusAddr)) + } if config.OpenCollectorEndpoint != "" { @@ -346,6 +348,8 @@ func (s *Server) setupMetrics(config *TelemetryConfig, serviceName string) error // set the tracer s.tracer = tracerProvider + + log.Info("Open collector tracing started", "address", config.OpenCollectorEndpoint) } return nil diff --git a/internal/cli/server/testdata/test.toml b/internal/cli/server/testdata/test.toml index 6d07abdad7..4ccc644ee9 100644 --- a/internal/cli/server/testdata/test.toml +++ b/internal/cli/server/testdata/test.toml @@ -1,7 +1,9 @@ datadir = "./data" +snapshot = false ["eth.requiredblocks"] -a = "b" +"31000000" = "0x2087b9e2b353209c2c21e370c82daa12278efd0fe5f0febe6c29035352cf050e" +"32000000" = "0x875500011e5eecc0c554f95d07b31cf59df4ca2505f4dbbfffa7d4e4da917c68" [p2p] maxpeers = 30 @@ -11,7 +13,7 @@ locals = [] lifetime = "1s" [miner] -mine = false +mine = true gaslimit = 30000000 gasprice = "1000000000" diff --git a/scripts/getconfig.go b/scripts/getconfig.go index 9b5e6bf2b3..689ed68fbf 100644 --- a/scripts/getconfig.go +++ b/scripts/getconfig.go @@ -445,27 +445,42 @@ func updateArgsAdd(args []string) []string { return args } -func handleGRPC(args []string, updatedArgs []string) []string { +func handlePrometheus(args []string, updatedArgs []string) []string { var newUpdatedArgs []string - var addr string + mAddr := "" + mPort := "" - var port string + pAddr := "" + pPort := "" newUpdatedArgs = append(newUpdatedArgs, updatedArgs...) for i, val := range args { - if strings.Contains(val, "pprof.addr") && strings.Contains(val, "-") { - addr = args[i+1] + if strings.Contains(val, "metrics.addr") && strings.HasPrefix(val, "-") { + mAddr = args[i+1] } - if strings.Contains(val, "pprof.port") && strings.Contains(val, "-") { - port = args[i+1] + if strings.Contains(val, "metrics.port") && strings.HasPrefix(val, "-") { + mPort = args[i+1] + } + + if strings.Contains(val, "pprof.addr") && strings.HasPrefix(val, "-") { + pAddr = args[i+1] + } + + if strings.Contains(val, "pprof.port") && strings.HasPrefix(val, "-") { + pPort = args[i+1] } } - newUpdatedArgs = append(newUpdatedArgs, "--grpc.addr") - newUpdatedArgs = append(newUpdatedArgs, addr+":"+port) + if mAddr != "" && mPort != "" { + newUpdatedArgs = append(newUpdatedArgs, "--metrics.prometheus-addr") + newUpdatedArgs = append(newUpdatedArgs, mAddr+":"+mPort) + } else if pAddr != "" && pPort != "" { + newUpdatedArgs = append(newUpdatedArgs, "--metrics.prometheus-addr") + newUpdatedArgs = append(newUpdatedArgs, pAddr+":"+pPort) + } return newUpdatedArgs } @@ -658,7 +673,7 @@ func main() { outOfDateFlags := checkFlag(allFlags, flagsToCheck) updatedArgs := updateArgsClean(args, outOfDateFlags) updatedArgs = updateArgsAdd(updatedArgs) - updatedArgs = handleGRPC(args, updatedArgs) + updatedArgs = handlePrometheus(args, updatedArgs) if temp == notYet { updatedArgs = append(updatedArgs, ignoreForNow...)