From a75c0610b7aad63de01cefeeaf3426b332e718eb Mon Sep 17 00:00:00 2001 From: meowsbits <45600330+meowsbits@users.noreply.github.com> Date: Mon, 23 Mar 2020 09:05:15 -0500 Subject: [PATCH 1/7] core/blockchain: simplify atomic store after writeBlockWithState (#20798) Signed-off-by: meows --- core/blockchain.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index de0d4f3993..1da899ab43 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1740,11 +1740,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // Write the block to the chain and get the status. substart = time.Now() status, err := bc.writeBlockWithState(block, receipts, logs, statedb, false) + atomic.StoreUint32(&followupInterrupt, 1) if err != nil { - atomic.StoreUint32(&followupInterrupt, 1) return it.index, err } - atomic.StoreUint32(&followupInterrupt, 1) // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them From 0734c4b82050fd8114bb18514191cb5c7ccf5259 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 23 Mar 2020 16:26:56 +0100 Subject: [PATCH 2/7] node, cmd/clef: report actual port used for http rpc (#20789) --- cmd/clef/main.go | 4 ++-- node/node.go | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index b2c8812ab2..dd898871d5 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -545,12 +545,12 @@ func signer(c *cli.Context) error { if err != nil { utils.Fatalf("Could not start RPC api: %v", err) } - extapiURL = fmt.Sprintf("http://%s", httpEndpoint) + extapiURL = fmt.Sprintf("http://%v/", listener.Addr()) log.Info("HTTP endpoint opened", "url", extapiURL) defer func() { listener.Close() - log.Info("HTTP endpoint closed", "url", httpEndpoint) + log.Info("HTTP endpoint closed", "url", extapiURL) }() } if !c.GlobalBool(utils.IPCDisabledFlag.Name) { diff --git a/node/node.go b/node/node.go index 1bdd5af7d2..7d8f9b07b9 100644 --- a/node/node.go +++ b/node/node.go @@ -368,7 +368,9 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors if err != nil { return err } - n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%s", endpoint), "cors", strings.Join(cors, ","), "vhosts", strings.Join(vhosts, ",")) + n.log.Info("HTTP endpoint opened", "url", fmt.Sprintf("http://%v/", listener.Addr()), + "cors", strings.Join(cors, ","), + "vhosts", strings.Join(vhosts, ",")) // All listeners booted successfully n.httpEndpoint = endpoint n.httpListener = listener @@ -380,10 +382,10 @@ func (n *Node) startHTTP(endpoint string, apis []rpc.API, modules []string, cors // stopHTTP terminates the HTTP RPC endpoint. func (n *Node) stopHTTP() { if n.httpListener != nil { + url := fmt.Sprintf("http://%v/", n.httpListener.Addr()) n.httpListener.Close() n.httpListener = nil - - n.log.Info("HTTP endpoint closed", "url", fmt.Sprintf("http://%s", n.httpEndpoint)) + n.log.Info("HTTP endpoint closed", "url", url) } if n.httpHandler != nil { n.httpHandler.Stop() From 39f502329fac4640cfb71959c3496f19ea88bc85 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 23 Mar 2020 18:21:23 +0100 Subject: [PATCH 3/7] internal/ethapi: don't set sender-balance to maxuint, fixes #16999 (#20783) Prior to this change, eth_call changed the balance of the sender account in the EVM environment to 2^256 wei to cover the gas cost of the call execution. We've had this behavior for a long time even though it's super confusing. This commit sets the default call gasprice to zero instead of updating the balance, which is better because it makes eth_call semantics less surprising. Removing the built-in balance assignment also makes balance overrides work as expected. --- eth/api_backend.go | 2 -- internal/ethapi/api.go | 6 +----- les/api_backend.go | 2 -- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index 0ad2d57fd2..a82aee4eb7 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" @@ -191,7 +190,6 @@ func (b *EthAPIBackend) GetTd(blockHash common.Hash) *big.Int { } func (b *EthAPIBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { - state.SetBalance(msg.From(), math.MaxBig256) vmError := func() error { return nil } context := core.NewEVMContext(msg, header, b.eth.BlockChain(), nil) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index cb4d255daf..68afa83c76 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -47,10 +47,6 @@ import ( "github.com/tyler-smith/go-bip39" ) -const ( - defaultGasPrice = params.GWei -) - // PublicEthereumAPI provides an API to access Ethereum related information. // It offers only methods that operate on public data that is freely available to anyone. type PublicEthereumAPI struct { @@ -808,7 +804,7 @@ func DoCall(ctx context.Context, b Backend, args CallArgs, blockNrOrHash rpc.Blo log.Warn("Caller gas above allowance, capping", "requested", gas, "cap", globalGasCap) gas = globalGasCap.Uint64() } - gasPrice := new(big.Int).SetUint64(defaultGasPrice) + gasPrice := new(big.Int) if args.GasPrice != nil { gasPrice = args.GasPrice.ToInt() } diff --git a/les/api_backend.go b/les/api_backend.go index 5627c34d6a..756beaf6a7 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" @@ -168,7 +167,6 @@ func (b *LesApiBackend) GetTd(hash common.Hash) *big.Int { } func (b *LesApiBackend) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header) (*vm.EVM, func() error, error) { - state.SetBalance(msg.From(), math.MaxBig256) context := core.NewEVMContext(msg, header, b.eth.blockchain, nil) return vm.NewEVM(context, state, b.eth.chainConfig, vm.Config{}), state.Error, nil } From 42e02ac03bfc4e01496d15314e94f46260d98f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 26 Mar 2020 11:24:01 +0200 Subject: [PATCH 4/7] metrics: disable CPU stats (gosigar) on iOS --- metrics/cpu.go | 12 ------------ metrics/cpu_disabled.go | 23 +++++++++++++++++++++++ metrics/cpu_enabled.go | 31 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 metrics/cpu_disabled.go create mode 100644 metrics/cpu_enabled.go diff --git a/metrics/cpu.go b/metrics/cpu.go index 3278d81616..72ece16e07 100644 --- a/metrics/cpu.go +++ b/metrics/cpu.go @@ -16,21 +16,9 @@ package metrics -import "github.com/elastic/gosigar" - // CPUStats is the system and process CPU stats. type CPUStats struct { GlobalTime int64 // Time spent by the CPU working on all processes GlobalWait int64 // Time spent by waiting on disk for all processes LocalTime int64 // Time spent by the CPU working on this process } - -// ReadCPUStats retrieves the current CPU stats. -func ReadCPUStats(stats *CPUStats) { - global := gosigar.Cpu{} - global.Get() - - stats.GlobalTime = int64(global.User + global.Nice + global.Sys) - stats.GlobalWait = int64(global.Wait) - stats.LocalTime = getProcessCPUTime() -} diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go new file mode 100644 index 0000000000..6c3428993f --- /dev/null +++ b/metrics/cpu_disabled.go @@ -0,0 +1,23 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build ios + +package metrics + +// ReadCPUStats retrieves the current CPU stats. Internally this uses `gosigar`, +// which is not supported on the platforms in this file. +func ReadCPUStats(stats *CPUStats) {} diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go new file mode 100644 index 0000000000..99d44e4002 --- /dev/null +++ b/metrics/cpu_enabled.go @@ -0,0 +1,31 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// +build !ios + +package metrics + +import "github.com/elastic/gosigar" + +// ReadCPUStats retrieves the current CPU stats. +func ReadCPUStats(stats *CPUStats) { + global := gosigar.Cpu{} + global.Get() + + stats.GlobalTime = int64(global.User + global.Nice + global.Sys) + stats.GlobalWait = int64(global.Wait) + stats.LocalTime = getProcessCPUTime() +} From 1583e7d2744af796aa85159217e80ca146097302 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 26 Mar 2020 12:51:50 +0100 Subject: [PATCH 5/7] cmd/devp2p: tweak DNS TTLs (#20801) * cmd/devp2p: tweak DNS TTLs * cmd/devp2p: bump treeNodeTTL to four weeks --- cmd/devp2p/dnscmd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 7c9ccd31f4..8ca31504d3 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -97,8 +97,8 @@ var ( ) const ( - rootTTL = 1 - treeNodeTTL = 2147483647 + rootTTL = 30 * 60 // 30 min + treeNodeTTL = 4 * 7 * 24 * 60 * 60 // 4 weeks ) // dnsSync performs dnsSyncCommand. From 87a411b839dc770460cd152cf2e7e6d40729709a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 26 Mar 2020 16:39:56 +0100 Subject: [PATCH 6/7] cmd/devp2p: lower route53 change limit again (#20819) --- cmd/devp2p/dns_route53.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 71118be543..d83446a73f 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -32,9 +32,11 @@ import ( "gopkg.in/urfave/cli.v1" ) -// The Route53 limits change sets to this size. DNS changes need to be split -// up into multiple batches to work around the limit. -const route53ChangeLimit = 30000 +// Route53 limits change sets to 32k of 'RDATA size'. DNS changes need to be split up into +// multiple batches to work around the limit. Unfortunately I cannot find any +// documentation explaining how the RDATA size of a change set is computed and the best we +// can do is estimate it. For this reason, our internal limit is much lower than 32k. +const route53ChangeLimit = 20000 var ( route53AccessKeyFlag = cli.StringFlag{ From d3c1e654f06a38cb1d78cab0503d872e4ca56bec Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 26 Mar 2020 23:55:33 +0100 Subject: [PATCH 7/7] cmd/devp2p: be very correct about route53 change splitting (#20820) Turns out the way RDATA limits work is documented after all, I just didn't search right. The trick to make it work is to count UPSERTs twice. This also adds an additional check to ensure TTL changes are applied on existing records. --- cmd/devp2p/dns_route53.go | 41 ++++++++++++++++++++++++---------- cmd/devp2p/dns_route53_test.go | 16 +++++++++++-- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index d83446a73f..c5f99529b8 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -32,11 +32,13 @@ import ( "gopkg.in/urfave/cli.v1" ) -// Route53 limits change sets to 32k of 'RDATA size'. DNS changes need to be split up into -// multiple batches to work around the limit. Unfortunately I cannot find any -// documentation explaining how the RDATA size of a change set is computed and the best we -// can do is estimate it. For this reason, our internal limit is much lower than 32k. -const route53ChangeLimit = 20000 +const ( + // Route53 limits change sets to 32k of 'RDATA size'. Change sets are also limited to + // 1000 items. UPSERTs count double. + // https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/DNSLimitations.html#limits-api-requests-changeresourcerecordsets + route53ChangeSizeLimit = 32000 + route53ChangeCountLimit = 1000 +) var ( route53AccessKeyFlag = cli.StringFlag{ @@ -104,7 +106,7 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { } // Submit change batches. - batches := splitChanges(changes, route53ChangeLimit) + batches := splitChanges(changes, route53ChangeSizeLimit, route53ChangeCountLimit) for i, changes := range batches { log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) batch := new(route53.ChangeBatch) @@ -178,7 +180,7 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e // Entry is unknown, push a new one log.Info(fmt.Sprintf("Creating %s = %q", path, val)) changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val))) - } else if prevValue != val { + } else if prevValue != val || prevRecords.ttl != ttl { // Entry already exists, only change its content. log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val)) changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val))) @@ -214,18 +216,26 @@ func sortChanges(changes []*route53.Change) { // splitChanges splits up DNS changes such that each change batch // is smaller than the given RDATA limit. -func splitChanges(changes []*route53.Change, limit int) [][]*route53.Change { - var batches [][]*route53.Change - var batchSize int +func splitChanges(changes []*route53.Change, sizeLimit, countLimit int) [][]*route53.Change { + var ( + batches [][]*route53.Change + batchSize int + batchCount int + ) for _, ch := range changes { // Start new batch if this change pushes the current one over the limit. - size := changeSize(ch) - if len(batches) == 0 || batchSize+size > limit { + count := changeCount(ch) + size := changeSize(ch) * count + overSize := batchSize+size > sizeLimit + overCount := batchCount+count > countLimit + if len(batches) == 0 || overSize || overCount { batches = append(batches, nil) batchSize = 0 + batchCount = 0 } batches[len(batches)-1] = append(batches[len(batches)-1], ch) batchSize += size + batchCount += count } return batches } @@ -241,6 +251,13 @@ func changeSize(ch *route53.Change) int { return size } +func changeCount(ch *route53.Change) int { + if *ch.Action == "UPSERT" { + return 2 + } + return 1 +} + // collectRecords collects all TXT records below the given name. func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) { log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID)) diff --git a/cmd/devp2p/dns_route53_test.go b/cmd/devp2p/dns_route53_test.go index f6ab6c1762..a2ef3791f6 100644 --- a/cmd/devp2p/dns_route53_test.go +++ b/cmd/devp2p/dns_route53_test.go @@ -140,11 +140,23 @@ func TestRoute53ChangeSort(t *testing.T) { t.Fatalf("wrong changes (got %d, want %d)", len(changes), len(wantChanges)) } + // Check splitting according to size. wantSplit := [][]*route53.Change{ wantChanges[:4], - wantChanges[4:8], + wantChanges[4:6], + wantChanges[6:], } - split := splitChanges(changes, 600) + split := splitChanges(changes, 600, 4000) + if !reflect.DeepEqual(split, wantSplit) { + t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit)) + } + + // Check splitting according to count. + wantSplit = [][]*route53.Change{ + wantChanges[:5], + wantChanges[5:], + } + split = splitChanges(changes, 10000, 6) if !reflect.DeepEqual(split, wantSplit) { t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit)) }