From c9e0b3105baa68e58780340cbaf146b1f0b46636 Mon Sep 17 00:00:00 2001 From: Chris Ziogas Date: Mon, 3 Jun 2024 13:30:27 +0300 Subject: [PATCH 01/19] Supply delta live tracer (#29347) Introduces the first built-in live tracer. The supply tracer tracks ETH supply changes across blocks and writes the output to disk. This will need to be enabled through CLI using the `--vmtrace supply` flag. Co-authored-by: Sina Mahmoodi --- .../gen_balance_change_reason_stringer.go | 37 ++ core/tracing/hooks.go | 2 + eth/tracers/internal/tracetest/supply_test.go | 613 ++++++++++++++++++ eth/tracers/live/gen_supplyinfoburn.go | 49 ++ eth/tracers/live/gen_supplyinfoissuance.go | 49 ++ eth/tracers/live/supply.go | 310 +++++++++ 6 files changed, 1060 insertions(+) create mode 100644 core/tracing/gen_balance_change_reason_stringer.go create mode 100644 eth/tracers/internal/tracetest/supply_test.go create mode 100644 eth/tracers/live/gen_supplyinfoburn.go create mode 100644 eth/tracers/live/gen_supplyinfoissuance.go create mode 100644 eth/tracers/live/supply.go diff --git a/core/tracing/gen_balance_change_reason_stringer.go b/core/tracing/gen_balance_change_reason_stringer.go new file mode 100644 index 0000000000..d3a515a12d --- /dev/null +++ b/core/tracing/gen_balance_change_reason_stringer.go @@ -0,0 +1,37 @@ +// Code generated by "stringer -type=BalanceChangeReason -output gen_balance_change_reason_stringer.go"; DO NOT EDIT. + +package tracing + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[BalanceChangeUnspecified-0] + _ = x[BalanceIncreaseRewardMineUncle-1] + _ = x[BalanceIncreaseRewardMineBlock-2] + _ = x[BalanceIncreaseWithdrawal-3] + _ = x[BalanceIncreaseGenesisBalance-4] + _ = x[BalanceIncreaseRewardTransactionFee-5] + _ = x[BalanceDecreaseGasBuy-6] + _ = x[BalanceIncreaseGasReturn-7] + _ = x[BalanceIncreaseDaoContract-8] + _ = x[BalanceDecreaseDaoAccount-9] + _ = x[BalanceChangeTransfer-10] + _ = x[BalanceChangeTouchAccount-11] + _ = x[BalanceIncreaseSelfdestruct-12] + _ = x[BalanceDecreaseSelfdestruct-13] + _ = x[BalanceDecreaseSelfdestructBurn-14] +} + +const _BalanceChangeReason_name = "BalanceChangeUnspecifiedBalanceIncreaseRewardMineUncleBalanceIncreaseRewardMineBlockBalanceIncreaseWithdrawalBalanceIncreaseGenesisBalanceBalanceIncreaseRewardTransactionFeeBalanceDecreaseGasBuyBalanceIncreaseGasReturnBalanceIncreaseDaoContractBalanceDecreaseDaoAccountBalanceChangeTransferBalanceChangeTouchAccountBalanceIncreaseSelfdestructBalanceDecreaseSelfdestructBalanceDecreaseSelfdestructBurn" + +var _BalanceChangeReason_index = [...]uint16{0, 24, 54, 84, 109, 138, 173, 194, 218, 244, 269, 290, 315, 342, 369, 400} + +func (i BalanceChangeReason) String() string { + if i >= BalanceChangeReason(len(_BalanceChangeReason_index)-1) { + return "BalanceChangeReason(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _BalanceChangeReason_name[_BalanceChangeReason_index[i]:_BalanceChangeReason_index[i+1]] +} diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 9b08cffd45..db058e847c 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -199,6 +199,8 @@ type Hooks struct { // for tracing and reporting. type BalanceChangeReason byte +//go:generate stringer -type=BalanceChangeReason -output gen_balance_change_reason_stringer.go + const ( BalanceChangeUnspecified BalanceChangeReason = 0 diff --git a/eth/tracers/internal/tracetest/supply_test.go b/eth/tracers/internal/tracetest/supply_test.go new file mode 100644 index 0000000000..2d4f1b0890 --- /dev/null +++ b/eth/tracers/internal/tracetest/supply_test.go @@ -0,0 +1,613 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package tracetest + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "math/big" + "os" + "path" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/params" + + // Force-load live packages, to trigger registration + _ "github.com/ethereum/go-ethereum/eth/tracers/live" +) + +type supplyInfoIssuance struct { + GenesisAlloc *hexutil.Big `json:"genesisAlloc,omitempty"` + Reward *hexutil.Big `json:"reward,omitempty"` + Withdrawals *hexutil.Big `json:"withdrawals,omitempty"` +} + +type supplyInfoBurn struct { + EIP1559 *hexutil.Big `json:"1559,omitempty"` + Blob *hexutil.Big `json:"blob,omitempty"` + Misc *hexutil.Big `json:"misc,omitempty"` +} + +type supplyInfo struct { + Issuance *supplyInfoIssuance `json:"issuance,omitempty"` + Burn *supplyInfoBurn `json:"burn,omitempty"` + + // Block info + Number uint64 `json:"blockNumber"` + Hash common.Hash `json:"hash"` + ParentHash common.Hash `json:"parentHash"` +} + +func emptyBlockGenerationFunc(b *core.BlockGen) {} + +func TestSupplyOmittedFields(t *testing.T) { + var ( + config = *params.MergedTestChainConfig + gspec = &core.Genesis{ + Config: &config, + } + ) + + gspec.Config.TerminalTotalDifficulty = big.NewInt(0) + + out, _, err := testSupplyTracer(t, gspec, func(b *core.BlockGen) { + b.SetPoS() + }) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + + expected := supplyInfo{ + Number: 0, + Hash: common.HexToHash("0x52f276d96f0afaaf2c3cb358868bdc2779c4b0cb8de3e7e5302e247c0b66a703"), + ParentHash: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), + } + actual := out[expected.Number] + + compareAsJSON(t, expected, actual) +} + +func TestSupplyGenesisAlloc(t *testing.T) { + var ( + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + eth1 = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + + config = *params.AllEthashProtocolChanges + + gspec = &core.Genesis{ + Config: &config, + Alloc: types.GenesisAlloc{ + addr1: {Balance: eth1}, + addr2: {Balance: eth1}, + }, + } + ) + + expected := supplyInfo{ + Issuance: &supplyInfoIssuance{ + GenesisAlloc: (*hexutil.Big)(new(big.Int).Mul(common.Big2, big.NewInt(params.Ether))), + }, + Number: 0, + Hash: common.HexToHash("0xbcc9466e9fc6a8b56f4b29ca353a421ff8b51a0c1a58ca4743b427605b08f2ca"), + ParentHash: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), + } + + out, _, err := testSupplyTracer(t, gspec, emptyBlockGenerationFunc) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + + actual := out[expected.Number] + + compareAsJSON(t, expected, actual) +} + +func TestSupplyRewards(t *testing.T) { + var ( + config = *params.AllEthashProtocolChanges + + gspec = &core.Genesis{ + Config: &config, + } + ) + + expected := supplyInfo{ + Issuance: &supplyInfoIssuance{ + Reward: (*hexutil.Big)(new(big.Int).Mul(common.Big2, big.NewInt(params.Ether))), + }, + Number: 1, + Hash: common.HexToHash("0xcbb08370505be503dafedc4e96d139ea27aba3cbc580148568b8a307b3f51052"), + ParentHash: common.HexToHash("0xadeda0a83e337b6c073e3f0e9a17531a04009b397a9588c093b628f21b8bc5a3"), + } + + out, _, err := testSupplyTracer(t, gspec, emptyBlockGenerationFunc) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + + actual := out[expected.Number] + + compareAsJSON(t, expected, actual) +} + +func TestSupplyEip1559Burn(t *testing.T) { + var ( + config = *params.AllEthashProtocolChanges + + aa = common.HexToAddress("0x000000000000000000000000000000000000aaaa") + // A sender who makes transactions, has some eth1 + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + gwei5 = new(big.Int).Mul(big.NewInt(5), big.NewInt(params.GWei)) + eth1 = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + + gspec = &core.Genesis{ + Config: &config, + BaseFee: big.NewInt(params.InitialBaseFee), + Alloc: types.GenesisAlloc{ + addr1: {Balance: eth1}, + }, + } + ) + + signer := types.LatestSigner(gspec.Config) + + eip1559BlockGenerationFunc := func(b *core.BlockGen) { + txdata := &types.DynamicFeeTx{ + ChainID: gspec.Config.ChainID, + Nonce: 0, + To: &aa, + Gas: 21000, + GasFeeCap: gwei5, + GasTipCap: big.NewInt(2), + } + tx := types.NewTx(txdata) + tx, _ = types.SignTx(tx, signer, key1) + + b.AddTx(tx) + } + + out, chain, err := testSupplyTracer(t, gspec, eip1559BlockGenerationFunc) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + var ( + head = chain.CurrentBlock() + reward = new(big.Int).Mul(common.Big2, big.NewInt(params.Ether)) + burn = new(big.Int).Mul(big.NewInt(21000), head.BaseFee) + expected = supplyInfo{ + Issuance: &supplyInfoIssuance{ + Reward: (*hexutil.Big)(reward), + }, + Burn: &supplyInfoBurn{ + EIP1559: (*hexutil.Big)(burn), + }, + Number: 1, + Hash: head.Hash(), + ParentHash: head.ParentHash, + } + ) + + actual := out[expected.Number] + compareAsJSON(t, expected, actual) +} + +func TestSupplyWithdrawals(t *testing.T) { + var ( + config = *params.MergedTestChainConfig + gspec = &core.Genesis{ + Config: &config, + } + ) + + withdrawalsBlockGenerationFunc := func(b *core.BlockGen) { + b.SetPoS() + + b.AddWithdrawal(&types.Withdrawal{ + Validator: 42, + Address: common.Address{0xee}, + Amount: 1337, + }) + } + + out, chain, err := testSupplyTracer(t, gspec, withdrawalsBlockGenerationFunc) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + + var ( + head = chain.CurrentBlock() + expected = supplyInfo{ + Issuance: &supplyInfoIssuance{ + Withdrawals: (*hexutil.Big)(big.NewInt(1337000000000)), + }, + Number: 1, + Hash: head.Hash(), + ParentHash: head.ParentHash, + } + actual = out[expected.Number] + ) + + compareAsJSON(t, expected, actual) +} + +// Tests fund retrieval after contract's selfdestruct. +// Contract A calls contract B which selfdestructs, but B receives eth1 +// after the selfdestruct opcode executes from Contract A. +// Because Contract B is removed only at the end of the transaction +// the ether sent in between is burnt before Cancun hard fork. +func TestSupplySelfdestruct(t *testing.T) { + var ( + config = *params.TestChainConfig + + aa = common.HexToAddress("0x1111111111111111111111111111111111111111") + bb = common.HexToAddress("0x2222222222222222222222222222222222222222") + dad = common.HexToAddress("0x0000000000000000000000000000000000000dad") + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + gwei5 = new(big.Int).Mul(big.NewInt(5), big.NewInt(params.GWei)) + eth1 = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + + gspec = &core.Genesis{ + Config: &config, + BaseFee: big.NewInt(params.InitialBaseFee), + Alloc: types.GenesisAlloc{ + addr1: {Balance: eth1}, + aa: { + Code: common.FromHex("0x61face60f01b6000527322222222222222222222222222222222222222226000806002600080855af160008103603457600080fd5b60008060008034865af1905060008103604c57600080fd5b5050"), + // Nonce: 0, + Balance: big.NewInt(0), + }, + bb: { + Code: common.FromHex("0x6000357fface000000000000000000000000000000000000000000000000000000000000808203602f57610dad80ff5b5050"), + Nonce: 0, + Balance: eth1, + }, + }, + } + ) + + gspec.Config.TerminalTotalDifficulty = big.NewInt(0) + + signer := types.LatestSigner(gspec.Config) + + testBlockGenerationFunc := func(b *core.BlockGen) { + b.SetPoS() + + txdata := &types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: gwei5, + Gas: 150000, + GasPrice: gwei5, + Data: []byte{}, + } + + tx := types.NewTx(txdata) + tx, _ = types.SignTx(tx, signer, key1) + + b.AddTx(tx) + } + + // 1. Test pre Cancun + preCancunOutput, preCancunChain, err := testSupplyTracer(t, gspec, testBlockGenerationFunc) + if err != nil { + t.Fatalf("Pre-cancun failed to test supply tracer: %v", err) + } + + // Check balance at state: + // 1. 0x0000...000dad has 1 ether + // 2. A has 0 ether + // 3. B has 0 ether + statedb, _ := preCancunChain.State() + if got, exp := statedb.GetBalance(dad), eth1; got.CmpBig(exp) != 0 { + t.Fatalf("Pre-cancun address \"%v\" balance, got %v exp %v\n", dad, got, exp) + } + if got, exp := statedb.GetBalance(aa), big.NewInt(0); got.CmpBig(exp) != 0 { + t.Fatalf("Pre-cancun address \"%v\" balance, got %v exp %v\n", aa, got, exp) + } + if got, exp := statedb.GetBalance(bb), big.NewInt(0); got.CmpBig(exp) != 0 { + t.Fatalf("Pre-cancun address \"%v\" balance, got %v exp %v\n", bb, got, exp) + } + + head := preCancunChain.CurrentBlock() + // Check live trace output + expected := supplyInfo{ + Burn: &supplyInfoBurn{ + EIP1559: (*hexutil.Big)(big.NewInt(55289500000000)), + Misc: (*hexutil.Big)(big.NewInt(5000000000)), + }, + Number: 1, + Hash: head.Hash(), + ParentHash: head.ParentHash, + } + + actual := preCancunOutput[expected.Number] + + compareAsJSON(t, expected, actual) + + // 2. Test post Cancun + cancunTime := uint64(0) + gspec.Config.ShanghaiTime = &cancunTime + gspec.Config.CancunTime = &cancunTime + + postCancunOutput, postCancunChain, err := testSupplyTracer(t, gspec, testBlockGenerationFunc) + if err != nil { + t.Fatalf("Post-cancun failed to test supply tracer: %v", err) + } + + // Check balance at state: + // 1. 0x0000...000dad has 1 ether + // 3. A has 0 ether + // 3. B has 5 gwei + statedb, _ = postCancunChain.State() + if got, exp := statedb.GetBalance(dad), eth1; got.CmpBig(exp) != 0 { + t.Fatalf("Post-shanghai address \"%v\" balance, got %v exp %v\n", dad, got, exp) + } + if got, exp := statedb.GetBalance(aa), big.NewInt(0); got.CmpBig(exp) != 0 { + t.Fatalf("Post-shanghai address \"%v\" balance, got %v exp %v\n", aa, got, exp) + } + if got, exp := statedb.GetBalance(bb), gwei5; got.CmpBig(exp) != 0 { + t.Fatalf("Post-shanghai address \"%v\" balance, got %v exp %v\n", bb, got, exp) + } + + // Check live trace output + head = postCancunChain.CurrentBlock() + expected = supplyInfo{ + Burn: &supplyInfoBurn{ + EIP1559: (*hexutil.Big)(big.NewInt(55289500000000)), + }, + Number: 1, + Hash: head.Hash(), + ParentHash: head.ParentHash, + } + + actual = postCancunOutput[expected.Number] + + compareAsJSON(t, expected, actual) +} + +// Tests selfdestructing contract to send its balance to itself (burn). +// It tests both cases of selfdestructing succeding and being reverted. +// - Contract A calls B and D. +// - Contract B selfdestructs and sends the eth1 to itself (Burn amount to be counted). +// - Contract C selfdestructs and sends the eth1 to itself. +// - Contract D calls C and reverts (Burn amount of C +// has to be reverted as well). +func TestSupplySelfdestructItselfAndRevert(t *testing.T) { + var ( + config = *params.TestChainConfig + + aa = common.HexToAddress("0x1111111111111111111111111111111111111111") + bb = common.HexToAddress("0x2222222222222222222222222222222222222222") + cc = common.HexToAddress("0x3333333333333333333333333333333333333333") + dd = common.HexToAddress("0x4444444444444444444444444444444444444444") + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + gwei5 = new(big.Int).Mul(big.NewInt(5), big.NewInt(params.GWei)) + eth1 = new(big.Int).Mul(common.Big1, big.NewInt(params.Ether)) + eth2 = new(big.Int).Mul(common.Big2, big.NewInt(params.Ether)) + eth5 = new(big.Int).Mul(big.NewInt(5), big.NewInt(params.Ether)) + + gspec = &core.Genesis{ + Config: &config, + // BaseFee: big.NewInt(params.InitialBaseFee), + Alloc: types.GenesisAlloc{ + addr1: {Balance: eth1}, + aa: { + // Contract code in YUL: + // + // object "ContractA" { + // code { + // let B := 0x2222222222222222222222222222222222222222 + // let D := 0x4444444444444444444444444444444444444444 + + // // Call to Contract B + // let resB:= call(gas(), B, 0, 0x0, 0x0, 0, 0) + + // // Call to Contract D + // let resD := call(gas(), D, 0, 0x0, 0x0, 0, 0) + // } + // } + Code: common.FromHex("0x73222222222222222222222222222222222222222273444444444444444444444444444444444444444460006000600060006000865af160006000600060006000865af150505050"), + Balance: common.Big0, + }, + bb: { + // Contract code in YUL: + // + // object "ContractB" { + // code { + // let self := address() + // selfdestruct(self) + // } + // } + Code: common.FromHex("0x3080ff50"), + Balance: eth5, + }, + cc: { + Code: common.FromHex("0x3080ff50"), + Balance: eth1, + }, + dd: { + // Contract code in YUL: + // + // object "ContractD" { + // code { + // let C := 0x3333333333333333333333333333333333333333 + + // // Call to Contract C + // let resC := call(gas(), C, 0, 0x0, 0x0, 0, 0) + + // // Revert + // revert(0, 0) + // } + // } + Code: common.FromHex("0x73333333333333333333333333333333333333333360006000600060006000855af160006000fd5050"), + Balance: eth2, + }, + }, + } + ) + + gspec.Config.TerminalTotalDifficulty = big.NewInt(0) + + signer := types.LatestSigner(gspec.Config) + + testBlockGenerationFunc := func(b *core.BlockGen) { + b.SetPoS() + + txdata := &types.LegacyTx{ + Nonce: 0, + To: &aa, + Value: common.Big0, + Gas: 150000, + GasPrice: gwei5, + Data: []byte{}, + } + + tx := types.NewTx(txdata) + tx, _ = types.SignTx(tx, signer, key1) + + b.AddTx(tx) + } + + output, chain, err := testSupplyTracer(t, gspec, testBlockGenerationFunc) + if err != nil { + t.Fatalf("failed to test supply tracer: %v", err) + } + + // Check balance at state: + // 1. A has 0 ether + // 2. B has 0 ether, burned + // 3. C has 2 ether, selfdestructed but parent D reverted + // 4. D has 1 ether, reverted + statedb, _ := chain.State() + if got, exp := statedb.GetBalance(aa), common.Big0; got.CmpBig(exp) != 0 { + t.Fatalf("address \"%v\" balance, got %v exp %v\n", aa, got, exp) + } + if got, exp := statedb.GetBalance(bb), common.Big0; got.CmpBig(exp) != 0 { + t.Fatalf("address \"%v\" balance, got %v exp %v\n", bb, got, exp) + } + if got, exp := statedb.GetBalance(cc), eth1; got.CmpBig(exp) != 0 { + t.Fatalf("address \"%v\" balance, got %v exp %v\n", bb, got, exp) + } + if got, exp := statedb.GetBalance(dd), eth2; got.CmpBig(exp) != 0 { + t.Fatalf("address \"%v\" balance, got %v exp %v\n", bb, got, exp) + } + + // Check live trace output + block := chain.GetBlockByNumber(1) + + expected := supplyInfo{ + Burn: &supplyInfoBurn{ + EIP1559: (*hexutil.Big)(new(big.Int).Mul(block.BaseFee(), big.NewInt(int64(block.GasUsed())))), + Misc: (*hexutil.Big)(eth5), // 5ETH burned from contract B + }, + Number: 1, + Hash: block.Hash(), + ParentHash: block.ParentHash(), + } + + actual := output[expected.Number] + + compareAsJSON(t, expected, actual) +} + +func testSupplyTracer(t *testing.T, genesis *core.Genesis, gen func(*core.BlockGen)) ([]supplyInfo, *core.BlockChain, error) { + var ( + engine = beacon.New(ethash.NewFaker()) + ) + + traceOutputPath := filepath.ToSlash(t.TempDir()) + traceOutputFilename := path.Join(traceOutputPath, "supply.jsonl") + + // Load supply tracer + tracer, err := tracers.LiveDirectory.New("supply", json.RawMessage(fmt.Sprintf(`{"path":"%s"}`, traceOutputPath))) + if err != nil { + return nil, nil, fmt.Errorf("failed to create call tracer: %v", err) + } + + chain, err := core.NewBlockChain(rawdb.NewMemoryDatabase(), core.DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, engine, vm.Config{Tracer: tracer}, nil, nil) + if err != nil { + return nil, nil, fmt.Errorf("failed to create tester chain: %v", err) + } + defer chain.Stop() + + _, blocks, _ := core.GenerateChainWithGenesis(genesis, engine, 1, func(i int, b *core.BlockGen) { + b.SetCoinbase(common.Address{1}) + gen(b) + }) + + if n, err := chain.InsertChain(blocks); err != nil { + return nil, chain, fmt.Errorf("block %d: failed to insert into chain: %v", n, err) + } + + // Check and compare the results + file, err := os.OpenFile(traceOutputFilename, os.O_RDONLY, 0666) + if err != nil { + return nil, chain, fmt.Errorf("failed to open output file: %v", err) + } + defer file.Close() + + var output []supplyInfo + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + blockBytes := scanner.Bytes() + + var info supplyInfo + if err := json.Unmarshal(blockBytes, &info); err != nil { + return nil, chain, fmt.Errorf("failed to unmarshal result: %v", err) + } + + output = append(output, info) + } + + return output, chain, nil +} + +func compareAsJSON(t *testing.T, expected interface{}, actual interface{}) { + want, err := json.Marshal(expected) + if err != nil { + t.Fatalf("failed to marshal expected value to JSON: %v", err) + } + + have, err := json.Marshal(actual) + if err != nil { + t.Fatalf("failed to marshal actual value to JSON: %v", err) + } + + if !bytes.Equal(want, have) { + t.Fatalf("incorrect supply info: expected %s, got %s", string(want), string(have)) + } +} diff --git a/eth/tracers/live/gen_supplyinfoburn.go b/eth/tracers/live/gen_supplyinfoburn.go new file mode 100644 index 0000000000..d01eda3975 --- /dev/null +++ b/eth/tracers/live/gen_supplyinfoburn.go @@ -0,0 +1,49 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package live + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*supplyInfoBurnMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s supplyInfoBurn) MarshalJSON() ([]byte, error) { + type supplyInfoBurn struct { + EIP1559 *hexutil.Big `json:"1559,omitempty"` + Blob *hexutil.Big `json:"blob,omitempty"` + Misc *hexutil.Big `json:"misc,omitempty"` + } + var enc supplyInfoBurn + enc.EIP1559 = (*hexutil.Big)(s.EIP1559) + enc.Blob = (*hexutil.Big)(s.Blob) + enc.Misc = (*hexutil.Big)(s.Misc) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *supplyInfoBurn) UnmarshalJSON(input []byte) error { + type supplyInfoBurn struct { + EIP1559 *hexutil.Big `json:"1559,omitempty"` + Blob *hexutil.Big `json:"blob,omitempty"` + Misc *hexutil.Big `json:"misc,omitempty"` + } + var dec supplyInfoBurn + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.EIP1559 != nil { + s.EIP1559 = (*big.Int)(dec.EIP1559) + } + if dec.Blob != nil { + s.Blob = (*big.Int)(dec.Blob) + } + if dec.Misc != nil { + s.Misc = (*big.Int)(dec.Misc) + } + return nil +} diff --git a/eth/tracers/live/gen_supplyinfoissuance.go b/eth/tracers/live/gen_supplyinfoissuance.go new file mode 100644 index 0000000000..e2536ee325 --- /dev/null +++ b/eth/tracers/live/gen_supplyinfoissuance.go @@ -0,0 +1,49 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package live + +import ( + "encoding/json" + "math/big" + + "github.com/ethereum/go-ethereum/common/hexutil" +) + +var _ = (*supplyInfoIssuanceMarshaling)(nil) + +// MarshalJSON marshals as JSON. +func (s supplyInfoIssuance) MarshalJSON() ([]byte, error) { + type supplyInfoIssuance struct { + GenesisAlloc *hexutil.Big `json:"genesisAlloc,omitempty"` + Reward *hexutil.Big `json:"reward,omitempty"` + Withdrawals *hexutil.Big `json:"withdrawals,omitempty"` + } + var enc supplyInfoIssuance + enc.GenesisAlloc = (*hexutil.Big)(s.GenesisAlloc) + enc.Reward = (*hexutil.Big)(s.Reward) + enc.Withdrawals = (*hexutil.Big)(s.Withdrawals) + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (s *supplyInfoIssuance) UnmarshalJSON(input []byte) error { + type supplyInfoIssuance struct { + GenesisAlloc *hexutil.Big `json:"genesisAlloc,omitempty"` + Reward *hexutil.Big `json:"reward,omitempty"` + Withdrawals *hexutil.Big `json:"withdrawals,omitempty"` + } + var dec supplyInfoIssuance + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.GenesisAlloc != nil { + s.GenesisAlloc = (*big.Int)(dec.GenesisAlloc) + } + if dec.Reward != nil { + s.Reward = (*big.Int)(dec.Reward) + } + if dec.Withdrawals != nil { + s.Withdrawals = (*big.Int)(dec.Withdrawals) + } + return nil +} diff --git a/eth/tracers/live/supply.go b/eth/tracers/live/supply.go new file mode 100644 index 0000000000..0c9141e99d --- /dev/null +++ b/eth/tracers/live/supply.go @@ -0,0 +1,310 @@ +package live + +import ( + "encoding/json" + "errors" + "fmt" + "math/big" + "path/filepath" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" + "gopkg.in/natefinch/lumberjack.v2" +) + +func init() { + tracers.LiveDirectory.Register("supply", newSupply) +} + +type supplyInfoIssuance struct { + GenesisAlloc *big.Int `json:"genesisAlloc,omitempty"` + Reward *big.Int `json:"reward,omitempty"` + Withdrawals *big.Int `json:"withdrawals,omitempty"` +} + +//go:generate go run github.com/fjl/gencodec -type supplyInfoIssuance -field-override supplyInfoIssuanceMarshaling -out gen_supplyinfoissuance.go +type supplyInfoIssuanceMarshaling struct { + GenesisAlloc *hexutil.Big + Reward *hexutil.Big + Withdrawals *hexutil.Big +} + +type supplyInfoBurn struct { + EIP1559 *big.Int `json:"1559,omitempty"` + Blob *big.Int `json:"blob,omitempty"` + Misc *big.Int `json:"misc,omitempty"` +} + +//go:generate go run github.com/fjl/gencodec -type supplyInfoBurn -field-override supplyInfoBurnMarshaling -out gen_supplyinfoburn.go +type supplyInfoBurnMarshaling struct { + EIP1559 *hexutil.Big + Blob *hexutil.Big + Misc *hexutil.Big +} + +type supplyInfo struct { + Issuance *supplyInfoIssuance `json:"issuance,omitempty"` + Burn *supplyInfoBurn `json:"burn,omitempty"` + + // Block info + Number uint64 `json:"blockNumber"` + Hash common.Hash `json:"hash"` + ParentHash common.Hash `json:"parentHash"` +} + +type supplyTxCallstack struct { + calls []supplyTxCallstack + burn *big.Int +} + +type supply struct { + delta supplyInfo + txCallstack []supplyTxCallstack // Callstack for current transaction + logger *lumberjack.Logger +} + +type supplyTracerConfig struct { + Path string `json:"path"` // Path to the directory where the tracer logs will be stored + MaxSize int `json:"maxSize"` // MaxSize is the maximum size in megabytes of the tracer log file before it gets rotated. It defaults to 100 megabytes. +} + +func newSupply(cfg json.RawMessage) (*tracing.Hooks, error) { + var config supplyTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, fmt.Errorf("failed to parse config: %v", err) + } + } + if config.Path == "" { + return nil, errors.New("supply tracer output path is required") + } + + // Store traces in a rotating file + logger := &lumberjack.Logger{ + Filename: filepath.Join(config.Path, "supply.jsonl"), + } + if config.MaxSize > 0 { + logger.MaxSize = config.MaxSize + } + + t := &supply{ + delta: newSupplyInfo(), + logger: logger, + } + return &tracing.Hooks{ + OnBlockStart: t.OnBlockStart, + OnBlockEnd: t.OnBlockEnd, + OnGenesisBlock: t.OnGenesisBlock, + OnTxStart: t.OnTxStart, + OnBalanceChange: t.OnBalanceChange, + OnEnter: t.OnEnter, + OnExit: t.OnExit, + OnClose: t.OnClose, + }, nil +} + +func newSupplyInfo() supplyInfo { + return supplyInfo{ + Issuance: &supplyInfoIssuance{ + GenesisAlloc: big.NewInt(0), + Reward: big.NewInt(0), + Withdrawals: big.NewInt(0), + }, + Burn: &supplyInfoBurn{ + EIP1559: big.NewInt(0), + Blob: big.NewInt(0), + Misc: big.NewInt(0), + }, + + Number: 0, + Hash: common.Hash{}, + ParentHash: common.Hash{}, + } +} + +func (s *supply) resetDelta() { + s.delta = newSupplyInfo() +} + +func (s *supply) OnBlockStart(ev tracing.BlockEvent) { + s.resetDelta() + + s.delta.Number = ev.Block.NumberU64() + s.delta.Hash = ev.Block.Hash() + s.delta.ParentHash = ev.Block.ParentHash() + + // Calculate Burn for this block + if ev.Block.BaseFee() != nil { + burn := new(big.Int).Mul(new(big.Int).SetUint64(ev.Block.GasUsed()), ev.Block.BaseFee()) + s.delta.Burn.EIP1559 = burn + } + // Blob burnt gas + if blobGas := ev.Block.BlobGasUsed(); blobGas != nil && *blobGas > 0 && ev.Block.ExcessBlobGas() != nil { + var ( + excess = *ev.Block.ExcessBlobGas() + baseFee = eip4844.CalcBlobFee(excess) + burn = new(big.Int).Mul(new(big.Int).SetUint64(*blobGas), baseFee) + ) + s.delta.Burn.Blob = burn + } +} + +func (s *supply) OnBlockEnd(err error) { + s.write(s.delta) +} + +func (s *supply) OnGenesisBlock(b *types.Block, alloc types.GenesisAlloc) { + s.resetDelta() + + s.delta.Number = b.NumberU64() + s.delta.Hash = b.Hash() + s.delta.ParentHash = b.ParentHash() + + // Initialize supply with total allocation in genesis block + for _, account := range alloc { + s.delta.Issuance.GenesisAlloc.Add(s.delta.Issuance.GenesisAlloc, account.Balance) + } + + s.write(s.delta) +} + +func (s *supply) OnBalanceChange(a common.Address, prevBalance, newBalance *big.Int, reason tracing.BalanceChangeReason) { + diff := new(big.Int).Sub(newBalance, prevBalance) + + // NOTE: don't handle "BalanceIncreaseGenesisBalance" because it is handled in OnGenesisBlock + switch reason { + case tracing.BalanceIncreaseRewardMineUncle: + case tracing.BalanceIncreaseRewardMineBlock: + s.delta.Issuance.Reward.Add(s.delta.Issuance.Reward, diff) + case tracing.BalanceIncreaseWithdrawal: + s.delta.Issuance.Withdrawals.Add(s.delta.Issuance.Withdrawals, diff) + case tracing.BalanceDecreaseSelfdestructBurn: + // BalanceDecreaseSelfdestructBurn is non-reversible as it happens + // at the end of the transaction. + s.delta.Burn.Misc.Sub(s.delta.Burn.Misc, diff) + default: + return + } +} + +func (s *supply) OnTxStart(vm *tracing.VMContext, tx *types.Transaction, from common.Address) { + s.txCallstack = make([]supplyTxCallstack, 0, 1) +} + +// internalTxsHandler handles internal transactions burned amount +func (s *supply) internalTxsHandler(call *supplyTxCallstack) { + // Handle Burned amount + if call.burn != nil { + s.delta.Burn.Misc.Add(s.delta.Burn.Misc, call.burn) + } + + if len(call.calls) > 0 { + // Recursivelly handle internal calls + for _, call := range call.calls { + callCopy := call + s.internalTxsHandler(&callCopy) + } + } +} + +func (s *supply) OnEnter(depth int, typ byte, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + call := supplyTxCallstack{ + calls: make([]supplyTxCallstack, 0), + } + + // This is a special case of burned amount which has to be handled here + // which happens when type == selfdestruct and from == to. + if vm.OpCode(typ) == vm.SELFDESTRUCT && from == to && value.Cmp(common.Big0) == 1 { + call.burn = value + } + + // Append call to the callstack, so we can fill the details in CaptureExit + s.txCallstack = append(s.txCallstack, call) +} + +func (s *supply) OnExit(depth int, output []byte, gasUsed uint64, err error, reverted bool) { + if depth == 0 { + // No need to handle Burned amount if transaction is reverted + if !reverted { + s.internalTxsHandler(&s.txCallstack[0]) + } + return + } + + size := len(s.txCallstack) + if size <= 1 { + return + } + // Pop call + call := s.txCallstack[size-1] + s.txCallstack = s.txCallstack[:size-1] + size -= 1 + + // In case of a revert, we can drop the call and all its subcalls. + // Caution, that this has to happen after popping the call from the stack. + if reverted { + return + } + s.txCallstack[size-1].calls = append(s.txCallstack[size-1].calls, call) +} + +func (s *supply) OnClose() { + if err := s.logger.Close(); err != nil { + log.Warn("failed to close supply tracer log file", "error", err) + } +} + +func (s *supply) write(data any) { + supply, ok := data.(supplyInfo) + if !ok { + log.Warn("failed to cast supply tracer data on write to log file") + return + } + + // Remove empty fields + if supply.Issuance.GenesisAlloc.Sign() == 0 { + supply.Issuance.GenesisAlloc = nil + } + + if supply.Issuance.Reward.Sign() == 0 { + supply.Issuance.Reward = nil + } + + if supply.Issuance.Withdrawals.Sign() == 0 { + supply.Issuance.Withdrawals = nil + } + + if supply.Issuance.GenesisAlloc == nil && supply.Issuance.Reward == nil && supply.Issuance.Withdrawals == nil { + supply.Issuance = nil + } + + if supply.Burn.EIP1559.Sign() == 0 { + supply.Burn.EIP1559 = nil + } + + if supply.Burn.Blob.Sign() == 0 { + supply.Burn.Blob = nil + } + + if supply.Burn.Misc.Sign() == 0 { + supply.Burn.Misc = nil + } + + if supply.Burn.EIP1559 == nil && supply.Burn.Blob == nil && supply.Burn.Misc == nil { + supply.Burn = nil + } + + out, _ := json.Marshal(supply) + if _, err := s.logger.Write(out); err != nil { + log.Warn("failed to write to supply tracer log file", "error", err) + } + if _, err := s.logger.Write([]byte{'\n'}); err != nil { + log.Warn("failed to write to supply tracer log file", "error", err) + } +} From d38b88a5a10800e1632833e284c6500c240c012e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 3 Jun 2024 19:17:12 +0800 Subject: [PATCH 02/19] core/state: introduce stateupdate structure (#29530) * core/state: introduce stateUpate structure * core/state: remove outdated function description * core/state: address comments --- core/genesis_test.go | 4 +- core/state/state_object.go | 270 +++++++++++----------- core/state/statedb.go | 391 ++++++++++++++------------------ core/state/statedb_fuzz_test.go | 30 ++- core/state/stateupdate.go | 133 +++++++++++ 5 files changed, 462 insertions(+), 366 deletions(-) create mode 100644 core/state/stateupdate.go diff --git a/core/genesis_test.go b/core/genesis_test.go index 31401e214c..ab408327d4 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -304,7 +304,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.Hex2Bytes("14398d42be3394ff8d50681816a4b7bf8d8283306f577faba2d5bc57498de23b") + expected := common.FromHex("14398d42be3394ff8d50681816a4b7bf8d8283306f577faba2d5bc57498de23b") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) @@ -314,7 +314,7 @@ func TestVerkleGenesisCommit(t *testing.T) { triedb := triedb.NewDatabase(db, &triedb.Config{IsVerkle: true, PathDB: pathdb.Defaults}) block := genesis.MustCommit(db, triedb) if !bytes.Equal(block.Root().Bytes(), expected) { - t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) + t.Fatalf("invalid genesis state root, expected %x, got %x", expected, block.Root()) } // Test that the trie is verkle diff --git a/core/state/state_object.go b/core/state/state_object.go index 252dcdf2b0..b7a215bd17 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -19,9 +19,7 @@ package state import ( "bytes" "fmt" - "io" "maps" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -34,14 +32,6 @@ import ( "github.com/holiman/uint256" ) -// hasherPool holds a pool of hashers used by state objects during concurrent -// trie updates. -var hasherPool = sync.Pool{ - New: func() interface{} { - return crypto.NewKeccakState() - }, -} - type Storage map[common.Hash]common.Hash func (s Storage) Copy() Storage { @@ -65,9 +55,20 @@ type stateObject struct { trie Trie // storage trie, which becomes non-nil on first access code []byte // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites - pendingStorage Storage // Storage entries that need to be flushed to disk, at the end of an entire block - dirtyStorage Storage // Storage entries that have been modified in the current transaction execution, reset for every transaction + originStorage Storage // Storage entries that have been accessed within the current block + dirtyStorage Storage // Storage entries that have been modified within the current transaction + pendingStorage Storage // Storage entries that have been modified within the current block + + // uncommittedStorage tracks a set of storage entries that have been modified + // but not yet committed since the "last commit operation", along with their + // original values before mutation. + // + // Specifically, the commit will be performed after each transaction before + // the byzantium fork, therefore the map is already reset at the transaction + // boundary; however post the byzantium fork, the commit will only be performed + // at the end of block, this set essentially tracks all the modifications + // made within the block. + uncommittedStorage Storage // Cache flags. dirtyCode bool // true if the code was updated @@ -96,22 +97,18 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s acct = types.NewEmptyStateAccount() } return &stateObject{ - db: db, - address: address, - addrHash: crypto.Keccak256Hash(address[:]), - origin: origin, - data: *acct, - originStorage: make(Storage), - pendingStorage: make(Storage), - dirtyStorage: make(Storage), + db: db, + address: address, + addrHash: crypto.Keccak256Hash(address[:]), + origin: origin, + data: *acct, + originStorage: make(Storage), + dirtyStorage: make(Storage), + pendingStorage: make(Storage), + uncommittedStorage: make(Storage), } } -// EncodeRLP implements rlp.Encoder. -func (s *stateObject) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, &s.data) -} - func (s *stateObject) markSelfdestructed() { s.selfDestructed = true } @@ -160,7 +157,7 @@ func (s *stateObject) getPrefetchedTrie() Trie { return s.db.prefetcher.trie(s.addrHash, s.data.Root) } -// GetState retrieves a value from the account storage trie. +// GetState retrieves a value associated with the given storage key. func (s *stateObject) GetState(key common.Hash) common.Hash { value, _ := s.getState(key) return value @@ -177,7 +174,8 @@ func (s *stateObject) getState(key common.Hash) (common.Hash, common.Hash) { return origin, origin } -// GetCommittedState retrieves a value from the committed account storage trie. +// GetCommittedState retrieves the value associated with the specific key +// without any mutations caused in the current execution. func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // If we have a pending write or clean cached, return that if value, pending := s.pendingStorage[key]; pending { @@ -193,6 +191,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // have been handles via pendingStorage above. // 2) we don't have new values, and can deliver empty response back if _, destructed := s.db.stateObjectsDestruct[s.address]; destructed { + s.originStorage[key] = common.Hash{} // track the empty slot as origin value return common.Hash{} } // If no live objects are available, attempt to use snapshots @@ -272,17 +271,26 @@ func (s *stateObject) setState(key common.Hash, value common.Hash, origin common func (s *stateObject) finalise() { slotsToPrefetch := make([][]byte, 0, len(s.dirtyStorage)) for key, value := range s.dirtyStorage { - // If the slot is different from its original value, move it into the - // pending area to be committed at the end of the block (and prefetch - // the pathways). - if value != s.originStorage[key] { - s.pendingStorage[key] = value - slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure + if origin, exist := s.uncommittedStorage[key]; exist && origin == value { + // The slot is reverted to its original value, delete the entry + // to avoid thrashing the data structures. + delete(s.uncommittedStorage, key) + } else if exist { + // The slot is modified to another value and the slot has been + // tracked for commit, do nothing here. } else { - // Otherwise, the slot was reverted to its original value, remove it - // from the pending area to avoid thrashing the data structure. - delete(s.pendingStorage, key) + // The slot is different from its original value and hasn't been + // tracked for commit yet. + s.uncommittedStorage[key] = s.GetCommittedState(key) + slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure } + // Aggregate the dirty storage slots into the pending area. It might + // be possible that the value of tracked slot here is same with the + // one in originStorage (e.g. the slot was modified in tx_a and then + // modified back in tx_b). We can't blindly remove it from pending + // map as the dirty slot might have been committed already (before the + // byzantium fork) and entry is necessary to modify the value back. + s.pendingStorage[key] = value } if s.db.prefetcher != nil && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash { if err := s.db.prefetcher.prefetch(s.addrHash, s.data.Root, s.address, slotsToPrefetch); err != nil { @@ -308,7 +316,7 @@ func (s *stateObject) finalise() { // It assumes all the dirty storage slots have been finalized before. func (s *stateObject) updateTrie() (Trie, error) { // Short circuit if nothing changed, don't bother with hashing anything - if len(s.pendingStorage) == 0 { + if len(s.uncommittedStorage) == 0 { return s.trie, nil } // Retrieve a pretecher populated trie, or fall back to the database @@ -325,20 +333,8 @@ func (s *stateObject) updateTrie() (Trie, error) { return nil, err } } - - // The snapshot storage map for the object - var ( - storage map[common.Hash][]byte - origin map[common.Hash][]byte - ) - // Insert all the pending storage updates into the trie - usedStorage := make([][]byte, 0, len(s.pendingStorage)) - - hasher := hasherPool.Get().(crypto.KeccakState) - defer hasherPool.Put(hasher) - - // Perform trie updates before deletions. This prevents resolution of unnecessary trie nodes - // in circumstances similar to the following: + // Perform trie updates before deletions. This prevents resolution of unnecessary trie nodes + // in circumstances similar to the following: // // Consider nodes `A` and `B` who share the same full node parent `P` and have no other siblings. // During the execution of a block: @@ -347,21 +343,23 @@ func (s *stateObject) updateTrie() (Trie, error) { // If the deletion is handled first, then `P` would be left with only one child, thus collapsed // into a shortnode. This requires `B` to be resolved from disk. // Whereas if the created node is handled first, then the collapse is avoided, and `B` is not resolved. - var deletions []common.Hash - for key, value := range s.pendingStorage { + var ( + deletions []common.Hash + used = make([][]byte, 0, len(s.uncommittedStorage)) + ) + for key, origin := range s.uncommittedStorage { // Skip noop changes, persist actual changes - if value == s.originStorage[key] { + value, exist := s.pendingStorage[key] + if value == origin { + log.Error("Storage update was noop", "address", s.address, "slot", key) + continue + } + if !exist { + log.Error("Storage slot is not found in pending area", s.address, "slot", key) continue } - prev := s.originStorage[key] - s.originStorage[key] = value - - var encoded []byte // rlp-encoded value to be used by the snapshot if (value != common.Hash{}) { - // Encoding []byte cannot fail, ok to ignore the error. - trimmed := common.TrimLeftZeroes(value[:]) - encoded, _ = rlp.EncodeToBytes(trimmed) - if err := tr.UpdateStorage(s.address, key[:], trimmed); err != nil { + if err := tr.UpdateStorage(s.address, key[:], common.TrimLeftZeroes(value[:])); err != nil { s.db.setError(err) return nil, err } @@ -369,39 +367,8 @@ func (s *stateObject) updateTrie() (Trie, error) { } else { deletions = append(deletions, key) } - // Cache the mutated storage slots until commit - if storage == nil { - s.db.storagesLock.Lock() - if storage = s.db.storages[s.addrHash]; storage == nil { - storage = make(map[common.Hash][]byte) - s.db.storages[s.addrHash] = storage - } - s.db.storagesLock.Unlock() - } - khash := crypto.HashData(hasher, key[:]) - storage[khash] = encoded // encoded will be nil if it's deleted - - // Cache the original value of mutated storage slots - if origin == nil { - s.db.storagesLock.Lock() - if origin = s.db.storagesOrigin[s.address]; origin == nil { - origin = make(map[common.Hash][]byte) - s.db.storagesOrigin[s.address] = origin - } - s.db.storagesLock.Unlock() - } - // Track the original value of slot only if it's mutated first time - if _, ok := origin[khash]; !ok { - if prev == (common.Hash{}) { - origin[khash] = nil // nil if it was not present previously - } else { - // Encoding []byte cannot fail, ok to ignore the error. - b, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(prev[:])) - origin[khash] = b - } - } // Cache the items for preloading - usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure + used = append(used, common.CopyBytes(key[:])) // Copy needed for closure } for _, key := range deletions { if err := tr.DeleteStorage(s.address, key[:]); err != nil { @@ -410,15 +377,10 @@ func (s *stateObject) updateTrie() (Trie, error) { } s.db.StorageDeleted.Add(1) } - // If no slots were touched, issue a warning as we shouldn't have done all - // the above work in the first place - if len(usedStorage) == 0 { - log.Error("State object update was noop", "addr", s.address, "slots", len(s.pendingStorage)) - } if s.db.prefetcher != nil { - s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage) + s.db.prefetcher.used(s.addrHash, s.data.Root, used) } - s.pendingStorage = make(Storage) // reset pending map + s.uncommittedStorage = make(Storage) // empties the commit markers return tr, nil } @@ -434,30 +396,79 @@ func (s *stateObject) updateRoot() { s.data.Root = tr.Hash() } -// commit obtains a set of dirty storage trie nodes and updates the account data. -// The returned set can be nil if nothing to commit. This function assumes all -// storage mutations have already been flushed into trie by updateRoot. +// commitStorage overwrites the clean storage with the storage changes and +// fulfills the storage diffs into the given accountUpdate struct. +func (s *stateObject) commitStorage(op *accountUpdate) { + var ( + buf = crypto.NewKeccakState() + encode = func(val common.Hash) []byte { + if val == (common.Hash{}) { + return nil + } + blob, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(val[:])) + return blob + } + ) + for key, val := range s.pendingStorage { + // Skip the noop storage changes, it might be possible the value + // of tracked slot is same in originStorage and pendingStorage + // map, e.g. the storage slot is modified in tx_a and then reset + // back in tx_b. + if val == s.originStorage[key] { + continue + } + hash := crypto.HashData(buf, key[:]) + if op.storages == nil { + op.storages = make(map[common.Hash][]byte) + } + op.storages[hash] = encode(val) + if op.storagesOrigin == nil { + op.storagesOrigin = make(map[common.Hash][]byte) + } + op.storagesOrigin[hash] = encode(s.originStorage[key]) + + // Overwrite the clean value of storage slots + s.originStorage[key] = val + } + s.pendingStorage = make(Storage) +} + +// commit obtains the account changes (metadata, storage slots, code) caused by +// state execution along with the dirty storage trie nodes. // // Note, commit may run concurrently across all the state objects. Do not assume // thread-safe access to the statedb. -func (s *stateObject) commit() (*trienode.NodeSet, error) { - // Short circuit if trie is not even loaded, don't bother with committing anything - if s.trie == nil { - s.origin = s.data.Copy() - return nil, nil +func (s *stateObject) commit() (*accountUpdate, *trienode.NodeSet, error) { + // commit the account metadata changes + op := &accountUpdate{ + address: s.address, + data: types.SlimAccountRLP(s.data), + } + if s.origin != nil { + op.origin = types.SlimAccountRLP(*s.origin) + } + // commit the contract code if it's modified + if s.dirtyCode { + op.code = &contractCode{ + hash: common.BytesToHash(s.CodeHash()), + blob: s.code, + } + s.dirtyCode = false // reset the dirty flag + } + // Commit storage changes and the associated storage trie + s.commitStorage(op) + if len(op.storages) == 0 { + // nothing changed, don't bother to commit the trie + s.origin = s.data.Copy() + return op, nil, nil } - // The trie is currently in an open state and could potentially contain - // cached mutations. Call commit to acquire a set of nodes that have been - // modified, the set can be nil if nothing to commit. root, nodes, err := s.trie.Commit(false) if err != nil { - return nil, err + return nil, nil, err } s.data.Root = root - - // Update original account data after commit s.origin = s.data.Copy() - return nodes, nil + return op, nodes, nil } // AddBalance adds amount to s's balance. @@ -500,18 +511,19 @@ func (s *stateObject) setBalance(amount *uint256.Int) { func (s *stateObject) deepCopy(db *StateDB) *stateObject { obj := &stateObject{ - db: db, - address: s.address, - addrHash: s.addrHash, - origin: s.origin, - data: s.data, - code: s.code, - originStorage: s.originStorage.Copy(), - pendingStorage: s.pendingStorage.Copy(), - dirtyStorage: s.dirtyStorage.Copy(), - dirtyCode: s.dirtyCode, - selfDestructed: s.selfDestructed, - newContract: s.newContract, + db: db, + address: s.address, + addrHash: s.addrHash, + origin: s.origin, + data: s.data, + code: s.code, + originStorage: s.originStorage.Copy(), + pendingStorage: s.pendingStorage.Copy(), + dirtyStorage: s.dirtyStorage.Copy(), + uncommittedStorage: s.uncommittedStorage.Copy(), + dirtyCode: s.dirtyCode, + selfDestructed: s.selfDestructed, + newContract: s.newContract, } if s.trie != nil { obj.trie = db.db.CopyTrie(s.trie) diff --git a/core/state/statedb.go b/core/state/statedb.go index fe262d4695..659c500c3b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -18,6 +18,7 @@ package state import ( + "errors" "fmt" "maps" "math/big" @@ -95,15 +96,6 @@ type StateDB struct { // It will be updated when the Commit is called. originalRoot common.Hash - // These maps hold the state changes (including the corresponding - // original value) that occurred in this **block**. - accounts map[common.Hash][]byte // The mutated accounts in 'slim RLP' encoding - accountsOrigin map[common.Address][]byte // The original value of mutated accounts in 'slim RLP' encoding - - storages map[common.Hash]map[common.Hash][]byte // The mutated slots in prefix-zero trimmed rlp format - storagesOrigin map[common.Address]map[common.Hash][]byte // The original value of mutated slots in prefix-zero trimmed rlp format - storagesLock sync.Mutex // Mutex protecting the maps during concurrent updates/commits - // This map holds 'live' objects, which will get modified while // processing a state transition. stateObjects map[common.Address]*stateObject @@ -171,9 +163,6 @@ type StateDB struct { StorageUpdated atomic.Int64 AccountDeleted int StorageDeleted atomic.Int64 - - // Testing hooks - onCommit func(states *triestate.Set) // Hook invoked when commit is performed } // New creates a new state from a given trie. @@ -187,10 +176,6 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) trie: tr, originalRoot: root, snaps: snaps, - accounts: make(map[common.Hash][]byte), - storages: make(map[common.Hash]map[common.Hash][]byte), - accountsOrigin: make(map[common.Address][]byte), - storagesOrigin: make(map[common.Address]map[common.Hash][]byte), stateObjects: make(map[common.Address]*stateObject), stateObjectsDestruct: make(map[common.Address]*types.StateAccount), mutations: make(map[common.Address]*mutation), @@ -380,7 +365,7 @@ func (s *StateDB) GetCodeHash(addr common.Address) common.Hash { return common.Hash{} } -// GetState retrieves a value from the given account's storage trie. +// GetState retrieves the value associated with the specific key. func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) if stateObject != nil { @@ -389,7 +374,8 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { return common.Hash{} } -// GetCommittedState retrieves a value from the given account's committed storage trie. +// GetCommittedState retrieves the value associated with the specific key +// without any mutations caused in the current execution. func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) if stateObject != nil { @@ -557,22 +543,6 @@ func (s *StateDB) updateStateObject(obj *stateObject) { if obj.dirtyCode { s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) } - // Cache the data until commit. Note, this update mechanism is not symmetric - // to the deletion, because whereas it is enough to track account updates - // at commit time, deletions need tracking at transaction boundary level to - // ensure we capture state clearing. - s.accounts[obj.addrHash] = types.SlimAccountRLP(obj.data) - - // Track the original value of mutated account, nil means it was not present. - // Skip if it has been tracked (because updateStateObject may be called - // multiple times in a block). - if _, ok := s.accountsOrigin[obj.address]; !ok { - if obj.origin == nil { - s.accountsOrigin[obj.address] = nil - } else { - s.accountsOrigin[obj.address] = types.SlimAccountRLP(*obj.origin) - } - } } // deleteStateObject removes the given object from the state trie. @@ -691,10 +661,6 @@ func (s *StateDB) Copy() *StateDB { trie: s.db.CopyTrie(s.trie), hasher: crypto.NewKeccakState(), originalRoot: s.originalRoot, - accounts: copySet(s.accounts), - storages: copy2DSet(s.storages), - accountsOrigin: copySet(s.accountsOrigin), - storagesOrigin: copy2DSet(s.storagesOrigin), stateObjects: make(map[common.Address]*stateObject, len(s.stateObjects)), stateObjectsDestruct: maps.Clone(s.stateObjectsDestruct), mutations: make(map[common.Address]*mutation, len(s.mutations)), @@ -803,13 +769,6 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { if _, ok := s.stateObjectsDestruct[obj.address]; !ok { s.stateObjectsDestruct[obj.address] = obj.origin } - // Note, we can't do this only at the end of a block because multiple - // transactions within the same block might self destruct and then - // resurrect an account; but the snapshotter needs both events. - delete(s.accounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) - delete(s.storages, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) - delete(s.accountsOrigin, obj.address) // Clear out any previously updated account data (may be recreated via a resurrect) - delete(s.storagesOrigin, obj.address) // Clear out any previously updated storage data (may be recreated via a resurrect) } else { obj.finalise() s.markUpdate(addr) @@ -1020,10 +979,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r } // deleteStorage is designed to delete the storage trie of a designated account. -// It could potentially be terminated if the storage size is excessively large, -// potentially leading to an out-of-memory panic. The function will make an attempt -// to utilize an efficient strategy if the associated state snapshot is reachable; -// otherwise, it will resort to a less-efficient approach. +// The function will make an attempt to utilize an efficient strategy if the +// associated state snapshot is reachable; otherwise, it will resort to a less +// efficient approach. func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) { var ( start = time.Now() @@ -1058,75 +1016,61 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root } // handleDestruction processes all destruction markers and deletes the account -// and associated storage slots if necessary. There are four possible situations -// here: +// and associated storage slots if necessary. There are four potential scenarios +// as following: // -// - the account was not existent and be marked as destructed -// -// - the account was not existent and be marked as destructed, -// however, it's resurrected later in the same block. -// -// - the account was existent and be marked as destructed -// -// - the account was existent and be marked as destructed, -// however it's resurrected later in the same block. +// (a) the account was not existent and be marked as destructed +// (b) the account was not existent and be marked as destructed, +// however, it's resurrected later in the same block. +// (c) the account was existent and be marked as destructed +// (d) the account was existent and be marked as destructed, +// however it's resurrected later in the same block. // // In case (a), nothing needs be deleted, nil to nil transition can be ignored. -// // In case (b), nothing needs be deleted, nil is used as the original value for // newly created account and storages -// // In case (c), **original** account along with its storages should be deleted, // with their values be tracked as original value. -// // In case (d), **original** account along with its storages should be deleted, // with their values be tracked as original value. -func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error { - // Short circuit if geth is running with hash mode. This procedure can consume - // considerable time and storage deletion isn't supported in hash mode, thus - // preemptively avoiding unnecessary expenses. - if s.db.TrieDB().Scheme() == rawdb.HashScheme { - return nil - } +func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trienode.NodeSet, error) { + var ( + nodes []*trienode.NodeSet + buf = crypto.NewKeccakState() + deletes = make(map[common.Hash]*accountDelete) + ) for addr, prev := range s.stateObjectsDestruct { - // The original account was non-existing, and it's marked as destructed - // in the scope of block. It can be case (a) or (b). - // - for (a), skip it without doing anything. - // - for (b), track account's original value as nil. It may overwrite - // the data cached in s.accountsOrigin set by 'updateStateObject'. - addrHash := crypto.Keccak256Hash(addr[:]) + // The account was non-existent, and it's marked as destructed in the scope + // of block. It can be either case (a) or (b) and will be interpreted as + // null->null state transition. + // - for (a), skip it without doing anything + // - for (b), the resurrected account with nil as original will be handled afterwards if prev == nil { - if _, ok := s.accounts[addrHash]; ok { - s.accountsOrigin[addr] = nil // case (b) - } continue } - // It can overwrite the data in s.accountsOrigin set by 'updateStateObject'. - s.accountsOrigin[addr] = types.SlimAccountRLP(*prev) // case (c) or (d) + // The account was existent, it can be either case (c) or (d). + addrHash := crypto.HashData(buf, addr.Bytes()) + op := &accountDelete{ + address: addr, + origin: types.SlimAccountRLP(*prev), + } + deletes[addrHash] = op - // Short circuit if the storage was empty. + // Short circuit if the origin storage was empty. if prev.Root == types.EmptyRootHash { continue } - // Remove storage slots belong to the account. + // Remove storage slots belonging to the account. slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) if err != nil { - return fmt.Errorf("failed to delete storage, err: %w", err) - } - if s.storagesOrigin[addr] == nil { - s.storagesOrigin[addr] = slots - } else { - // It can overwrite the data in s.storagesOrigin[addrHash] set by - // 'object.updateTrie'. - for key, val := range slots { - s.storagesOrigin[addr][key] = val - } - } - if err := nodes.Merge(set); err != nil { - return err + return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err) } + op.storagesOrigin = slots + + // Aggregate the associated trie node changes. + nodes = append(nodes, set) } - return nil + return deletes, nodes, nil } // GetTrie returns the account trie. @@ -1134,18 +1078,12 @@ func (s *StateDB) GetTrie() Trie { return s.trie } -// Commit writes the state to the underlying in-memory trie database. -// Once the state is committed, tries cached in stateDB (including account -// trie, storage tries) will no longer be functional. A new state instance -// must be created with new root and updated database for accessing post- -// commit states. -// -// The associated block number of the state transition is also provided -// for more chain context. -func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, error) { +// commit gathers the state mutations accumulated along with the associated +// trie changes, resetting all internal flags with the new state as the base. +func (s *StateDB) commit(deleteEmptyObjects bool) (*stateUpdate, error) { // Short circuit in case any database failure occurred earlier. if s.dbErr != nil { - return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) + return nil, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) } // Finalize any pending changes and merge everything into the tries s.IntermediateRoot(deleteEmptyObjects) @@ -1156,19 +1094,56 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er accountTrieNodesDeleted int storageTrieNodesUpdated int storageTrieNodesDeleted int - nodes = trienode.NewMergedNodeSet() + + lock sync.Mutex // protect two maps below + nodes = trienode.NewMergedNodeSet() // aggregated trie nodes + updates = make(map[common.Hash]*accountUpdate, len(s.mutations)) // aggregated account updates + + // merge aggregates the dirty trie nodes into the global set. + // + // Given that some accounts may be destroyed and then recreated within + // the same block, it's possible that a node set with the same owner + // may already exists. In such cases, these two sets are combined, with + // the later one overwriting the previous one if any nodes are modified + // or deleted in both sets. + // + // merge run concurrently across all the state objects and account trie. + merge = func(set *trienode.NodeSet) error { + if set == nil { + return nil + } + lock.Lock() + defer lock.Unlock() + + updates, deletes := set.Size() + if set.Owner == (common.Hash{}) { + accountTrieNodesUpdated += updates + accountTrieNodesDeleted += deletes + } else { + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deletes + } + return nodes.Merge(set) + } ) - // Handle all state deletions first - if err := s.handleDestruction(nodes); err != nil { - return common.Hash{}, err + // Given that some accounts could be destroyed and then recreated within + // the same block, account deletions must be processed first. This ensures + // that the storage trie nodes deleted during destruction and recreated + // during subsequent resurrection can be combined correctly. + deletes, delNodes, err := s.handleDestruction() + if err != nil { + return nil, err + } + for _, set := range delNodes { + if err := merge(set); err != nil { + return nil, err + } } // Handle all state updates afterwards, concurrently to one another to shave // off some milliseconds from the commit operation. Also accumulate the code // writes to run in parallel with the computations. - start := time.Now() var ( - code = s.db.DiskDB().NewBatch() - lock sync.Mutex + start = time.Now() root common.Hash workers errgroup.Group ) @@ -1189,15 +1164,8 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } root = newroot - // Merge the dirty nodes of account trie into global set - lock.Lock() - defer lock.Unlock() - - if set != nil { - if err = nodes.Merge(set); err != nil { - return err - } - accountTrieNodesUpdated, accountTrieNodesDeleted = set.Size() + if err := merge(set); err != nil { + return err } s.AccountCommits = time.Since(start) return nil @@ -1215,49 +1183,29 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er } // Write any contract code associated with the state object obj := s.stateObjects[addr] - if obj.code != nil && obj.dirtyCode { - rawdb.WriteCode(code, common.BytesToHash(obj.CodeHash()), obj.code) - obj.dirtyCode = false + if obj == nil { + return nil, errors.New("missing state object") } // Run the storage updates concurrently to one another workers.Go(func() error { // Write any storage changes in the state object to its storage trie - set, err := obj.commit() + update, set, err := obj.commit() if err != nil { return err } - // Merge the dirty nodes of storage trie into global set. It is possible - // that the account was destructed and then resurrected in the same block. - // In this case, the node set is shared by both accounts. - lock.Lock() - defer lock.Unlock() - - if set != nil { - if err = nodes.Merge(set); err != nil { - return err - } - updates, deleted := set.Size() - storageTrieNodesUpdated += updates - storageTrieNodesDeleted += deleted + if err := merge(set); err != nil { + return err } + lock.Lock() + updates[obj.addrHash] = update + lock.Unlock() s.StorageCommits = time.Since(start) // overwrite with the longest storage commit runtime return nil }) } - // Schedule the code commits to run concurrently too. This shouldn't really - // take much since we don't often commit code, but since it's disk access, - // it's always yolo. - workers.Go(func() error { - if code.ValueSize() > 0 { - if err := code.Write(); err != nil { - log.Crit("Failed to commit dirty codes", "error", err) - } - } - return nil - }) // Wait for everything to finish and update the metrics if err := workers.Wait(); err != nil { - return common.Hash{}, err + return nil, err } accountUpdatedMeter.Mark(int64(s.AccountUpdated)) storageUpdatedMeter.Mark(s.StorageUpdated.Load()) @@ -1271,53 +1219,78 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er s.StorageUpdated.Store(0) s.StorageDeleted.Store(0) - // If snapshotting is enabled, update the snapshot tree with this new version - if s.snap != nil { - start = time.Now() - // Only update if there's a state transition (skip empty Clique blocks) - if parent := s.snap.Root(); parent != root { - if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { - log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) + // Clear all internal flags and update state root at the end. + s.mutations = make(map[common.Address]*mutation) + s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount) + + origin := s.originalRoot + s.originalRoot = root + return newStateUpdate(origin, root, deletes, updates, nodes), nil +} + +// commitAndFlush is a wrapper of commit which also commits the state mutations +// to the configured data stores. +func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool) (*stateUpdate, error) { + ret, err := s.commit(deleteEmptyObjects) + if err != nil { + return nil, err + } + // Commit dirty contract code if any exists + if db := s.db.DiskDB(); db != nil && len(ret.codes) > 0 { + batch := db.NewBatch() + for _, code := range ret.codes { + rawdb.WriteCode(batch, code.hash, code.blob) + } + if err := batch.Write(); err != nil { + return nil, err + } + } + if !ret.empty() { + // If snapshotting is enabled, update the snapshot tree with this new version + if s.snap != nil { + s.snap = nil + + start := time.Now() + if err := s.snaps.Update(ret.root, ret.originRoot, ret.destructs, ret.accounts, ret.storages); err != nil { + log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) } - // Keep TriesInMemory diff layers in the memory, persistent layer is 129th. + // Keep 128 diff layers in the memory, persistent layer is 129th. // - head layer is paired with HEAD state // - head-1 layer is paired with HEAD-1 state // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state - if err := s.snaps.Cap(root, TriesInMemory); err != nil { - log.Warn("Failed to cap snapshot tree", "root", root, "layers", TriesInMemory, "err", err) + if err := s.snaps.Cap(ret.root, TriesInMemory); err != nil { + log.Warn("Failed to cap snapshot tree", "root", ret.root, "layers", TriesInMemory, "err", err) } + s.SnapshotCommits += time.Since(start) } - s.SnapshotCommits += time.Since(start) - s.snap = nil - } - if root == (common.Hash{}) { - root = types.EmptyRootHash - } - origin := s.originalRoot - if origin == (common.Hash{}) { - origin = types.EmptyRootHash - } - if root != origin { - start = time.Now() - set := triestate.New(s.accountsOrigin, s.storagesOrigin) - if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { - return common.Hash{}, err + // If trie database is enabled, commit the state update as a new layer + if db := s.db.TrieDB(); db != nil { + start := time.Now() + set := triestate.New(ret.accountsOrigin, ret.storagesOrigin) + if err := db.Update(ret.root, ret.originRoot, block, ret.nodes, set); err != nil { + return nil, err + } + s.TrieDBCommits += time.Since(start) } - s.originalRoot = root - s.TrieDBCommits += time.Since(start) + } + return ret, err +} - if s.onCommit != nil { - s.onCommit(set) - } +// Commit writes the state mutations into the configured data stores. +// +// Once the state is committed, tries cached in stateDB (including account +// trie, storage tries) will no longer be functional. A new state instance +// must be created with new root and updated database for accessing post- +// commit states. +// +// The associated block number of the state transition is also provided +// for more chain context. +func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, error) { + ret, err := s.commitAndFlush(block, deleteEmptyObjects) + if err != nil { + return common.Hash{}, err } - // Clear all internal flags at the end of commit operation. - s.accounts = make(map[common.Hash][]byte) - s.storages = make(map[common.Hash]map[common.Hash][]byte) - s.accountsOrigin = make(map[common.Address][]byte) - s.storagesOrigin = make(map[common.Address]map[common.Hash][]byte) - s.mutations = make(map[common.Address]*mutation) - s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount) - return root, nil + return ret.root, nil } // Prepare handles the preparatory steps for executing a state transition with. @@ -1399,41 +1372,9 @@ func (s *StateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addre return s.accessList.Contains(addr, slot) } -// convertAccountSet converts a provided account set from address keyed to hash keyed. -func (s *StateDB) convertAccountSet(set map[common.Address]*types.StateAccount) map[common.Hash]struct{} { - ret := make(map[common.Hash]struct{}, len(set)) - for addr := range set { - obj, exist := s.stateObjects[addr] - if !exist { - ret[crypto.Keccak256Hash(addr[:])] = struct{}{} - } else { - ret[obj.addrHash] = struct{}{} - } - } - return ret -} - -// copySet returns a deep-copied set. -func copySet[k comparable](set map[k][]byte) map[k][]byte { - copied := make(map[k][]byte, len(set)) - for key, val := range set { - copied[key] = common.CopyBytes(val) - } - return copied -} - -// copy2DSet returns a two-dimensional deep-copied set. -func copy2DSet[k comparable](set map[k]map[common.Hash][]byte) map[k]map[common.Hash][]byte { - copied := make(map[k]map[common.Hash][]byte, len(set)) - for addr, subset := range set { - copied[addr] = make(map[common.Hash][]byte, len(subset)) - for key, val := range subset { - copied[addr][key] = common.CopyBytes(val) - } - } - return copied -} - +// markDelete is invoked when an account is deleted but the deletion is +// not yet committed. The pending mutation is cached and will be applied +// all together func (s *StateDB) markDelete(addr common.Address) { if _, ok := s.mutations[addr]; !ok { s.mutations[addr] = &mutation{} diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 6317681a7f..40b079cd8a 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/triestate" "github.com/ethereum/go-ethereum/triedb" "github.com/ethereum/go-ethereum/triedb/pathdb" "github.com/holiman/uint256" @@ -180,9 +179,21 @@ func (test *stateTest) run() bool { roots []common.Hash accountList []map[common.Address][]byte storageList []map[common.Address]map[common.Hash][]byte - onCommit = func(states *triestate.Set) { - accountList = append(accountList, copySet(states.Accounts)) - storageList = append(storageList, copy2DSet(states.Storages)) + copyUpdate = func(update *stateUpdate) { + accounts := make(map[common.Address][]byte, len(update.accountsOrigin)) + for key, val := range update.accountsOrigin { + accounts[key] = common.CopyBytes(val) + } + accountList = append(accountList, accounts) + + storages := make(map[common.Address]map[common.Hash][]byte, len(update.storagesOrigin)) + for addr, subset := range update.storagesOrigin { + storages[addr] = make(map[common.Hash][]byte, len(subset)) + for key, val := range subset { + storages[addr][key] = common.CopyBytes(val) + } + } + storageList = append(storageList, storages) } disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, &triedb.Config{PathDB: pathdb.Defaults}) @@ -210,8 +221,6 @@ func (test *stateTest) run() bool { if err != nil { panic(err) } - state.onCommit = onCommit - for i, action := range actions { if i%test.chunk == 0 && i != 0 { if byzantium { @@ -227,14 +236,15 @@ func (test *stateTest) run() bool { } else { state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary } - nroot, err := state.Commit(0, true) // call commit at the block boundary + ret, err := state.commitAndFlush(0, true) // call commit at the block boundary if err != nil { panic(err) } - if nroot == root { - return true // filter out non-change state transition + if ret.empty() { + return true } - roots = append(roots, nroot) + copyUpdate(ret) + roots = append(roots, ret.root) } for i := 0; i < len(test.actions); i++ { root := types.EmptyRootHash diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go new file mode 100644 index 0000000000..f3e6af997e --- /dev/null +++ b/core/state/stateupdate.go @@ -0,0 +1,133 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie/trienode" +) + +// contractCode represents a contract code with associated metadata. +type contractCode struct { + hash common.Hash // hash is the cryptographic hash of the contract code. + blob []byte // blob is the binary representation of the contract code. +} + +// accountDelete represents an operation for deleting an Ethereum account. +type accountDelete struct { + address common.Address // address is the unique account identifier + origin []byte // origin is the original value of account data in slim-RLP encoding. + storagesOrigin map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in prefix-zero-trimmed RLP format. +} + +// accountUpdate represents an operation for updating an Ethereum account. +type accountUpdate struct { + address common.Address // address is the unique account identifier + data []byte // data is the slim-RLP encoded account data. + origin []byte // origin is the original value of account data in slim-RLP encoding. + code *contractCode // code represents mutated contract code; nil means it's not modified. + storages map[common.Hash][]byte // storages stores mutated slots in prefix-zero-trimmed RLP format. + storagesOrigin map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in prefix-zero-trimmed RLP format. +} + +// stateUpdate represents the difference between two states resulting from state +// execution. It contains information about mutated contract codes, accounts, +// and storage slots, along with their original values. +type stateUpdate struct { + originRoot common.Hash // hash of the state before applying mutation + root common.Hash // hash of the state after applying mutation + destructs map[common.Hash]struct{} // destructs contains the list of destructed accounts + accounts map[common.Hash][]byte // accounts stores mutated accounts in 'slim RLP' encoding + accountsOrigin map[common.Address][]byte // accountsOrigin stores the original values of mutated accounts in 'slim RLP' encoding + storages map[common.Hash]map[common.Hash][]byte // storages stores mutated slots in 'prefix-zero-trimmed' RLP format + storagesOrigin map[common.Address]map[common.Hash][]byte // storagesOrigin stores the original values of mutated slots in 'prefix-zero-trimmed' RLP format + codes map[common.Address]contractCode // codes contains the set of dirty codes + nodes *trienode.MergedNodeSet // Aggregated dirty nodes caused by state changes +} + +// empty returns a flag indicating the state transition is empty or not. +func (sc *stateUpdate) empty() bool { + return sc.originRoot == sc.root +} + +// newStateUpdate constructs a state update object, representing the differences +// between two states by performing state execution. It aggregates the given +// account deletions and account updates to form a comprehensive state update. +func newStateUpdate(originRoot common.Hash, root common.Hash, deletes map[common.Hash]*accountDelete, updates map[common.Hash]*accountUpdate, nodes *trienode.MergedNodeSet) *stateUpdate { + var ( + destructs = make(map[common.Hash]struct{}) + accounts = make(map[common.Hash][]byte) + accountsOrigin = make(map[common.Address][]byte) + storages = make(map[common.Hash]map[common.Hash][]byte) + storagesOrigin = make(map[common.Address]map[common.Hash][]byte) + codes = make(map[common.Address]contractCode) + ) + // Due to the fact that some accounts could be destructed and resurrected + // within the same block, the deletions must be aggregated first. + for addrHash, op := range deletes { + addr := op.address + destructs[addrHash] = struct{}{} + accountsOrigin[addr] = op.origin + if len(op.storagesOrigin) > 0 { + storagesOrigin[addr] = op.storagesOrigin + } + } + // Aggregate account updates then. + for addrHash, op := range updates { + // Aggregate dirty contract codes if they are available. + addr := op.address + if op.code != nil { + codes[addr] = *op.code + } + // Aggregate the account changes. The original account value will only + // be tracked if it's not present yet. + accounts[addrHash] = op.data + if _, found := accountsOrigin[addr]; !found { + accountsOrigin[addr] = op.origin + } + // Aggregate the storage changes. The original storage slot value will + // only be tracked if it's not present yet. + if len(op.storages) > 0 { + storages[addrHash] = op.storages + } + if len(op.storagesOrigin) > 0 { + origin := storagesOrigin[addr] + if origin == nil { + storagesOrigin[addr] = op.storagesOrigin + continue + } + for key, slot := range op.storagesOrigin { + if _, found := origin[key]; !found { + origin[key] = slot + } + } + storagesOrigin[addr] = origin + } + } + return &stateUpdate{ + originRoot: types.TrieRootHash(originRoot), + root: types.TrieRootHash(root), + destructs: destructs, + accounts: accounts, + accountsOrigin: accountsOrigin, + storages: storages, + storagesOrigin: storagesOrigin, + codes: codes, + nodes: nodes, + } +} From 50405e29b73140f80cf2008be7827752a430a199 Mon Sep 17 00:00:00 2001 From: HAOYUatHZ <37070449+HAOYUatHZ@users.noreply.github.com> Date: Mon, 3 Jun 2024 20:31:23 +0800 Subject: [PATCH 03/19] cmd/evm/internal/t8ntool: fix a typo (#29887) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * i8ntool: fix a typo * cmd/evm/internal/t8ntool: fix typo typo --------- Co-authored-by: Péter Szilágyi --- cmd/evm/internal/t8ntool/transition.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 9ea94d195e..8fd068b2ea 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -217,7 +217,7 @@ func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error { return nil } if env.ParentBaseFee == nil || env.Number == 0 { - return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) + return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'parentBaseFee' in env section")) } env.BaseFee = eip1559.CalcBaseFee(chainConfig, &types.Header{ Number: new(big.Int).SetUint64(env.Number - 1), From b36c73813c6101c2cd5dfa227dc8e40b417e23ba Mon Sep 17 00:00:00 2001 From: maskpp Date: Mon, 3 Jun 2024 19:38:08 +0700 Subject: [PATCH 04/19] beacon/engine: prealloc capacity for map and slice (#29903) * prealloc capacity for map and slice * revert unnecessary change --- beacon/engine/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beacon/engine/types.go b/beacon/engine/types.go index a73691ca05..1dfcf5b71a 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -209,7 +209,7 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash, if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) } - var blobHashes []common.Hash + var blobHashes = make([]common.Hash, 0, len(txs)) for _, tx := range txs { blobHashes = append(blobHashes, tx.BlobHashes()...) } From 7270cba25c71d72447a535d0aafcd39892cf8112 Mon Sep 17 00:00:00 2001 From: miles <66052478+miles-six@users.noreply.github.com> Date: Mon, 3 Jun 2024 20:50:24 +0800 Subject: [PATCH 05/19] log: fix a typo (#29883) logger --- log/logger_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/log/logger_test.go b/log/logger_test.go index 6f415eb471..f1a9a93bce 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -26,7 +26,7 @@ func TestLoggingWithVmodule(t *testing.T) { logger.Trace("a message", "foo", "bar") have := out.String() // The timestamp is locale-dependent, so we want to trim that off - // "INFO [01-01|00:00:00.000] a messag ..." -> "a messag..." + // "INFO [01-01|00:00:00.000] a message ..." -> "a message..." have = strings.Split(have, "]")[1] want := " a message foo=bar\n" if have != want { @@ -42,7 +42,7 @@ func TestTerminalHandlerWithAttrs(t *testing.T) { logger.Trace("a message", "foo", "bar") have := out.String() // The timestamp is locale-dependent, so we want to trim that off - // "INFO [01-01|00:00:00.000] a messag ..." -> "a messag..." + // "INFO [01-01|00:00:00.000] a message ..." -> "a message..." have = strings.Split(have, "]")[1] want := " a message baz=bat foo=bar\n" if have != want { From a6751d6fc830b9a1edc963df2c05908f4392e39c Mon Sep 17 00:00:00 2001 From: tianyeyouyou <150894831+tianyeyouyou@users.noreply.github.com> Date: Mon, 3 Jun 2024 20:51:04 +0800 Subject: [PATCH 06/19] core/rawdb,eth/protocols,p2p: prealloc slice size (#29893) chore: prealloc slice size --- eth/protocols/snap/sync.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 53cba5d115..88d7d34dcc 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -3250,9 +3250,9 @@ func (t *healRequestSort) Merge() []TrieNodePathSet { // sortByAccountPath takes hashes and paths, and sorts them. After that, it generates // the TrieNodePaths and merges paths which belongs to the same account path. func sortByAccountPath(paths []string, hashes []common.Hash) ([]string, []common.Hash, []trie.SyncPath, []TrieNodePathSet) { - var syncPaths []trie.SyncPath - for _, path := range paths { - syncPaths = append(syncPaths, trie.NewSyncPath([]byte(path))) + syncPaths := make([]trie.SyncPath, len(paths)) + for i, path := range paths { + syncPaths[i] = trie.NewSyncPath([]byte(path)) } n := &healRequestSort{paths, hashes, syncPaths} sort.Sort(n) From adbbd8cd7b3c0d3e5a8475c3e3557f11ca38a3d1 Mon Sep 17 00:00:00 2001 From: Steven Wang <915337710@qq.com> Date: Tue, 4 Jun 2024 11:12:24 +0800 Subject: [PATCH 07/19] core/state: prefetch account trie while starting a prefetcher (#29919) Always prefetch the account trie while starting the prefetcher. Co-authored-by: steven Co-authored-by: rjl493456442 --- core/state/statedb.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/state/statedb.go b/core/state/statedb.go index 659c500c3b..bb2a87b172 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -208,6 +208,19 @@ func (s *StateDB) StartPrefetcher(namespace string) { } if s.snap != nil { s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace) + + // With the switch to the Proof-of-Stake consensus algorithm, block production + // rewards are now handled at the consensus layer. Consequently, a block may + // have no state transitions if it contains no transactions and no withdrawals. + // In such cases, the account trie won't be scheduled for prefetching, leading + // to unnecessary error logs. + // + // To prevent this, the account trie is always scheduled for prefetching once + // the prefetcher is constructed. For more details, see: + // https://github.com/ethereum/go-ethereum/issues/29880 + if err := s.prefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, nil); err != nil { + log.Error("Failed to prefetch account trie", "root", s.originalRoot, "err", err) + } } } From 68c0ec08158c2e61a40baa4489fd4377f6d1a223 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 4 Jun 2024 11:17:26 +0800 Subject: [PATCH 08/19] trie: iterate values pre-order and fix seek behavior (#27838) This pull request fixes the pre-order trie traversal by defining a more accurate iterator order and path comparison rule. Co-authored-by: Gary Rong --- trie/iterator.go | 81 ++++++++++++++++++++++++++++++++++--------- trie/iterator_test.go | 18 ++++++---- 2 files changed, 76 insertions(+), 23 deletions(-) diff --git a/trie/iterator.go b/trie/iterator.go index 83ccc0740f..fa01611063 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -135,7 +135,7 @@ type nodeIteratorState struct { node node // Trie node being iterated parent common.Hash // Hash of the first full ancestor node (nil if current is the root) index int // Child to be processed next - pathlen int // Length of the path to this node + pathlen int // Length of the path to the parent node } type nodeIterator struct { @@ -145,7 +145,7 @@ type nodeIterator struct { err error // Failure set in case of an internal error in the iterator resolver NodeResolver // optional node resolver for avoiding disk hits - pool []*nodeIteratorState // local pool for iteratorstates + pool []*nodeIteratorState // local pool for iterator states } // errIteratorEnd is stored in nodeIterator.err when iteration is done. @@ -304,6 +304,7 @@ func (it *nodeIterator) seek(prefix []byte) error { // The path we're looking for is the hex encoded key without terminator. key := keybytesToHex(prefix) key = key[:len(key)-1] + // Move forward until we're just before the closest match to key. for { state, parentIndex, path, err := it.peekSeek(key) @@ -311,7 +312,7 @@ func (it *nodeIterator) seek(prefix []byte) error { return errIteratorEnd } else if err != nil { return seekError{prefix, err} - } else if bytes.Compare(path, key) >= 0 { + } else if reachedPath(path, key) { return nil } it.push(state, parentIndex, path) @@ -339,7 +340,6 @@ func (it *nodeIterator) peek(descend bool) (*nodeIteratorState, *int, []byte, er // If we're skipping children, pop the current node first it.pop() } - // Continue iteration to the next child for len(it.stack) > 0 { parent := it.stack[len(it.stack)-1] @@ -372,7 +372,6 @@ func (it *nodeIterator) peekSeek(seekKey []byte) (*nodeIteratorState, *int, []by // If we're skipping children, pop the current node first it.pop() } - // Continue iteration to the next child for len(it.stack) > 0 { parent := it.stack[len(it.stack)-1] @@ -449,16 +448,18 @@ func (it *nodeIterator) findChild(n *fullNode, index int, ancestor common.Hash) state *nodeIteratorState childPath []byte ) - for ; index < len(n.Children); index++ { + for ; index < len(n.Children); index = nextChildIndex(index) { if n.Children[index] != nil { child = n.Children[index] hash, _ := child.cache() + state = it.getFromPool() state.hash = common.BytesToHash(hash) state.node = child state.parent = ancestor state.index = -1 state.pathlen = len(path) + childPath = append(childPath, path...) childPath = append(childPath, byte(index)) return child, state, childPath, index @@ -471,8 +472,8 @@ func (it *nodeIterator) nextChild(parent *nodeIteratorState, ancestor common.Has switch node := parent.node.(type) { case *fullNode: // Full node, move to the first non-nil child. - if child, state, path, index := it.findChild(node, parent.index+1, ancestor); child != nil { - parent.index = index - 1 + if child, state, path, index := it.findChild(node, nextChildIndex(parent.index), ancestor); child != nil { + parent.index = prevChildIndex(index) return state, path, true } case *shortNode: @@ -498,23 +499,23 @@ func (it *nodeIterator) nextChildAt(parent *nodeIteratorState, ancestor common.H switch n := parent.node.(type) { case *fullNode: // Full node, move to the first non-nil child before the desired key position - child, state, path, index := it.findChild(n, parent.index+1, ancestor) + child, state, path, index := it.findChild(n, nextChildIndex(parent.index), ancestor) if child == nil { // No more children in this fullnode return parent, it.path, false } // If the child we found is already past the seek position, just return it. - if bytes.Compare(path, key) >= 0 { - parent.index = index - 1 + if reachedPath(path, key) { + parent.index = prevChildIndex(index) return state, path, true } // The child is before the seek position. Try advancing for { - nextChild, nextState, nextPath, nextIndex := it.findChild(n, index+1, ancestor) + nextChild, nextState, nextPath, nextIndex := it.findChild(n, nextChildIndex(index), ancestor) // If we run out of children, or skipped past the target, return the // previous one - if nextChild == nil || bytes.Compare(nextPath, key) >= 0 { - parent.index = index - 1 + if nextChild == nil || reachedPath(nextPath, key) { + parent.index = prevChildIndex(index) return state, path, true } // We found a better child closer to the target @@ -541,7 +542,7 @@ func (it *nodeIterator) push(state *nodeIteratorState, parentIndex *int, path [] it.path = path it.stack = append(it.stack, state) if parentIndex != nil { - *parentIndex++ + *parentIndex = nextChildIndex(*parentIndex) } } @@ -550,8 +551,54 @@ func (it *nodeIterator) pop() { it.path = it.path[:last.pathlen] it.stack[len(it.stack)-1] = nil it.stack = it.stack[:len(it.stack)-1] - // last is now unused - it.putInPool(last) + + it.putInPool(last) // last is now unused +} + +// reachedPath normalizes a path by truncating a terminator if present, and +// returns true if it is greater than or equal to the target. Using this, +// the path of a value node embedded a full node will compare less than the +// full node's children. +func reachedPath(path, target []byte) bool { + if hasTerm(path) { + path = path[:len(path)-1] + } + return bytes.Compare(path, target) >= 0 +} + +// A value embedded in a full node occupies the last slot (16) of the array of +// children. In order to produce a pre-order traversal when iterating children, +// we jump to this last slot first, then go back iterate the child nodes (and +// skip the last slot at the end): + +// prevChildIndex returns the index of a child in a full node which precedes +// the given index when performing a pre-order traversal. +func prevChildIndex(index int) int { + switch index { + case 0: // We jumped back to iterate the children, from the value slot + return 16 + case 16: // We jumped to the embedded value slot at the end, from the placeholder index + return -1 + case 17: // We skipped the value slot after iterating all the children + return 15 + default: // We are iterating the children in sequence + return index - 1 + } +} + +// nextChildIndex returns the index of a child in a full node which follows +// the given index when performing a pre-order traversal. +func nextChildIndex(index int) int { + switch index { + case -1: // Jump from the placeholder index to the embedded value slot + return 16 + case 15: // Skip the value slot after iterating the children + return 17 + case 16: // From the embedded value slot, jump back to iterate the children + return 0 + default: // Iterate children in sequence + return index + 1 + } } func compareNodes(a, b NodeIterator) int { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 41e83f6cb6..a8f3c4413e 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -182,14 +182,14 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { type kvs struct{ k, v string } var testdata1 = []kvs{ + {"bar", "b"}, {"barb", "ba"}, {"bard", "bc"}, {"bars", "bb"}, - {"bar", "b"}, {"fab", "z"}, + {"foo", "a"}, {"food", "ab"}, {"foos", "aa"}, - {"foo", "a"}, } var testdata2 = []kvs{ @@ -218,7 +218,7 @@ func TestIteratorSeek(t *testing.T) { // Seek to a non-existent key. it = NewIterator(trie.MustNodeIterator([]byte("barc"))) - if err := checkIteratorOrder(testdata1[1:], it); err != nil { + if err := checkIteratorOrder(testdata1[2:], it); err != nil { t.Fatal(err) } @@ -227,6 +227,12 @@ func TestIteratorSeek(t *testing.T) { if err := checkIteratorOrder(nil, it); err != nil { t.Fatal(err) } + + // Seek to a key for which a prefixing key exists. + it = NewIterator(trie.MustNodeIterator([]byte("food"))) + if err := checkIteratorOrder(testdata1[6:], it); err != nil { + t.Fatal(err) + } } func checkIteratorOrder(want []kvs, it *Iterator) error { @@ -311,16 +317,16 @@ func TestUnionIterator(t *testing.T) { all := []struct{ k, v string }{ {"aardvark", "c"}, + {"bar", "b"}, {"barb", "ba"}, {"barb", "bd"}, {"bard", "bc"}, {"bars", "bb"}, {"bars", "be"}, - {"bar", "b"}, {"fab", "z"}, + {"foo", "a"}, {"food", "ab"}, {"foos", "aa"}, - {"foo", "a"}, {"jars", "d"}, } @@ -512,7 +518,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin rawdb.WriteTrieNode(diskdb, common.Hash{}, barNodePath, barNodeHash, barNodeBlob, triedb.Scheme()) } // Check that iteration produces the right set of values. - if err := checkIteratorOrder(testdata1[2:], NewIterator(it)); err != nil { + if err := checkIteratorOrder(testdata1[3:], NewIterator(it)); err != nil { t.Fatal(err) } } From 682ae838b2312a4ec8e5581069039b567e33c7c2 Mon Sep 17 00:00:00 2001 From: SangIlMo <156392700+SangIlMo@users.noreply.github.com> Date: Tue, 4 Jun 2024 16:59:41 +0900 Subject: [PATCH 09/19] internal/ethapi: recap higher args.Gas with block GasLimit in DoEstimateGas (#29738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * internal/ethapi: recap higher args.Gas with block GasLimit in DoEstimateGas * internal/ethapi: fix gas estimator capping code * internal/ethapi: fix test * fix goimports lint (remove space) --------- Co-authored-by: Péter Szilágyi --- internal/ethapi/api.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 8866a5b870..f211dcc659 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1199,10 +1199,16 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr State: state, ErrorRatio: estimateGasErrorRatio, } + // Set any required transaction default, but make sure the gas cap itself is not messed with + // if it was not specified in the original argument list. + if args.Gas == nil { + args.Gas = new(hexutil.Uint64) + } if err := args.CallDefaults(gasCap, header.BaseFee, b.ChainConfig().ChainID); err != nil { return 0, err } call := args.ToMessage(header.BaseFee) + // Run the gas estimation and wrap any revertals into a custom return estimate, revert, err := gasestimator.Estimate(ctx, call, opts, gasCap) if err != nil { From 125fb1ff5855134b040e295f380eeecff29af375 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 4 Jun 2024 20:51:34 +0800 Subject: [PATCH 10/19] core/state: avoid data race (#29924) --- core/state/statedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index bb2a87b172..61e76cdd77 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1211,8 +1211,8 @@ func (s *StateDB) commit(deleteEmptyObjects bool) (*stateUpdate, error) { } lock.Lock() updates[obj.addrHash] = update - lock.Unlock() s.StorageCommits = time.Since(start) // overwrite with the longest storage commit runtime + lock.Unlock() return nil }) } From 5550d8399f744aca0f50fc20bcd57bf2abeb13e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jun 2024 11:06:37 +0300 Subject: [PATCH 11/19] params: release Geth v1.14.4 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index a0e2de5a49..62140f05c7 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 14 // Minor version component of the current release - VersionPatch = 4 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 4 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From fc40d68e5bd6ffb1651a75f985114de5eabd4e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 5 Jun 2024 11:20:35 +0300 Subject: [PATCH 12/19] params: begin v1.14.5 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 62140f05c7..5ae1187b82 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 14 // Minor version component of the current release - VersionPatch = 4 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 14 // Minor version component of the current release + VersionPatch = 5 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From e85e21c93267f619c8ce75c0457f4ed52cd9bbf0 Mon Sep 17 00:00:00 2001 From: Hteev Oli Date: Wed, 5 Jun 2024 17:07:37 +0800 Subject: [PATCH 13/19] core/state, eth/tracers: fix typos (#29932) --- core/state/trie_prefetcher.go | 2 +- eth/tracers/internal/tracetest/supply_test.go | 2 +- eth/tracers/live/supply.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index ce94ab5139..5e5afbbecc 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -82,7 +82,7 @@ func (p *triePrefetcher) terminate(async bool) { return default: } - // Termiante all sub-fetchers, sync or async, depending on the request + // Terminate all sub-fetchers, sync or async, depending on the request for _, fetcher := range p.fetchers { fetcher.terminate(async) } diff --git a/eth/tracers/internal/tracetest/supply_test.go b/eth/tracers/internal/tracetest/supply_test.go index 2d4f1b0890..d608b1e002 100644 --- a/eth/tracers/internal/tracetest/supply_test.go +++ b/eth/tracers/internal/tracetest/supply_test.go @@ -398,7 +398,7 @@ func TestSupplySelfdestruct(t *testing.T) { } // Tests selfdestructing contract to send its balance to itself (burn). -// It tests both cases of selfdestructing succeding and being reverted. +// It tests both cases of selfdestructing succeeding and being reverted. // - Contract A calls B and D. // - Contract B selfdestructs and sends the eth1 to itself (Burn amount to be counted). // - Contract C selfdestructs and sends the eth1 to itself. diff --git a/eth/tracers/live/supply.go b/eth/tracers/live/supply.go index 0c9141e99d..936ffb9472 100644 --- a/eth/tracers/live/supply.go +++ b/eth/tracers/live/supply.go @@ -205,7 +205,7 @@ func (s *supply) internalTxsHandler(call *supplyTxCallstack) { } if len(call.calls) > 0 { - // Recursivelly handle internal calls + // Recursively handle internal calls for _, call := range call.calls { callCopy := call s.internalTxsHandler(&callCopy) From d09ddac39950645b95eae94894c322fc19f23acb Mon Sep 17 00:00:00 2001 From: Marquis Shanahan <29431502+9547@users.noreply.github.com> Date: Wed, 5 Jun 2024 21:05:00 +0800 Subject: [PATCH 14/19] core/rawdb: remove unused deriveLogFields (#29913) Co-authored-by: Gary Rong Co-authored-by: Martin HS --- core/rawdb/accessors_chain.go | 22 ---------------------- core/rawdb/accessors_chain_test.go | 6 ++---- 2 files changed, 2 insertions(+), 26 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 025be7ade7..c4735c850c 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -19,7 +19,6 @@ package rawdb import ( "bytes" "encoding/binary" - "errors" "fmt" "math/big" "slices" @@ -695,27 +694,6 @@ func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { return nil } -// deriveLogFields fills the logs in receiptLogs with information such as block number, txhash, etc. -func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, txs types.Transactions) error { - logIndex := uint(0) - if len(txs) != len(receipts) { - return errors.New("transaction and receipt count mismatch") - } - for i := 0; i < len(receipts); i++ { - txHash := txs[i].Hash() - // The derived log fields can simply be set from the block and transaction - for j := 0; j < len(receipts[i].Logs); j++ { - receipts[i].Logs[j].BlockNumber = number - receipts[i].Logs[j].BlockHash = hash - receipts[i].Logs[j].TxHash = txHash - receipts[i].Logs[j].TxIndex = uint(i) - receipts[i].Logs[j].Index = logIndex - logIndex++ - } - } - return nil -} - // ReadLogs retrieves the logs for all transactions in a block. In case // receipts is not found, a nil is returned. // Note: ReadLogs does not derive unstored log fields. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index fdc940b57e..2d30af4b3d 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -794,7 +794,7 @@ func TestDeriveLogFields(t *testing.T) { }), } // Create the corresponding receipts - receipts := []*receiptLogs{ + receipts := []*types.Receipt{ { Logs: []*types.Log{ {Address: common.BytesToAddress([]byte{0x11})}, @@ -818,9 +818,7 @@ func TestDeriveLogFields(t *testing.T) { // Derive log metadata fields number := big.NewInt(1) hash := common.BytesToHash([]byte{0x03, 0x14}) - if err := deriveLogFields(receipts, hash, number.Uint64(), txs); err != nil { - t.Fatal(err) - } + types.Receipts(receipts).DeriveFields(params.TestChainConfig, hash, number.Uint64(), 0, big.NewInt(0), big.NewInt(0), txs) // Iterate over all the computed fields and check that they're correct logIndex := uint(0) From bc6569462dfd78394599c93fd9e2a21908aca662 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 Jun 2024 19:31:04 +0200 Subject: [PATCH 15/19] p2p: use netip.Addr where possible (#29891) enode.Node was recently changed to store a cache of endpoint information. The IP address in the cache is a netip.Addr. I chose that type over net.IP because it is just better. netip.Addr is meant to be used as a value type. Copying it does not allocate, it can be compared with ==, and can be used as a map key. This PR changes most uses of Node.IP() into Node.IPAddr(), which returns the cached value directly without allocating. While there are still some public APIs left where net.IP is used, I have converted all code used internally by p2p/discover to the new types. So this does change some public Go API, but hopefully not APIs any external code actually uses. There weren't supposed to be any semantic differences resulting from this refactoring, however it does introduce one: In package p2p/netutil we treated the 0.0.0.0/8 network (addresses 0.x.y.z) as LAN, but netip.Addr.IsPrivate() doesn't. The treatment of this particular IP address range is controversial, with some software supporting it and others not. IANA lists it as special-purpose and invalid as a destination for a long time, so I don't know why I put it into the LAN list. It has now been marked as special in p2p/netutil as well. --- cmd/devp2p/internal/ethtest/conn.go | 3 +- cmd/devp2p/internal/v4test/framework.go | 6 +- cmd/devp2p/nodesetcmd.go | 6 +- cmd/devp2p/rlpxcmd.go | 6 +- p2p/dial.go | 27 ++-- p2p/discover/table.go | 52 ++++---- p2p/discover/table_test.go | 14 +-- p2p/discover/table_util_test.go | 5 +- p2p/discover/v4_udp.go | 33 ++--- p2p/discover/v4_udp_test.go | 8 +- p2p/discover/v5_udp.go | 12 +- p2p/discover/v5wire/encoding_test.go | 2 +- p2p/enode/localnode.go | 42 ++----- p2p/enode/localnode_test.go | 16 ++- p2p/enode/nodedb.go | 62 +++++---- p2p/enode/nodedb_test.go | 35 +++--- p2p/netutil/addrutil.go | 49 ++++++-- p2p/netutil/iptrack.go | 24 ++-- p2p/netutil/iptrack_test.go | 59 +++++---- p2p/netutil/net.go | 160 +++++++++++++----------- p2p/netutil/net_test.go | 64 +++++++--- p2p/server.go | 15 +-- p2p/server_nat_test.go | 5 +- 23 files changed, 392 insertions(+), 313 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index ba3c0585fd..757b137aa1 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -53,7 +53,8 @@ func (s *Suite) dial() (*Conn, error) { // dialAs attempts to dial a given node and perform a handshake using the given // private key. func (s *Suite) dialAs(key *ecdsa.PrivateKey) (*Conn, error) { - fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", s.Dest.IP(), s.Dest.TCP())) + tcpEndpoint, _ := s.Dest.TCPEndpoint() + fd, err := net.Dial("tcp", tcpEndpoint.String()) if err != nil { return nil, err } diff --git a/cmd/devp2p/internal/v4test/framework.go b/cmd/devp2p/internal/v4test/framework.go index df1f1f8aba..958fb71179 100644 --- a/cmd/devp2p/internal/v4test/framework.go +++ b/cmd/devp2p/internal/v4test/framework.go @@ -53,10 +53,12 @@ func newTestEnv(remote string, listen1, listen2 string) *testenv { if err != nil { panic(err) } - if node.IP() == nil || node.UDP() == 0 { + if !node.IPAddr().IsValid() || node.UDP() == 0 { var ip net.IP var tcpPort, udpPort int - if ip = node.IP(); ip == nil { + if node.IPAddr().IsValid() { + ip = node.IPAddr().AsSlice() + } else { ip = net.ParseIP("127.0.0.1") } if tcpPort = node.TCP(); tcpPort == 0 { diff --git a/cmd/devp2p/nodesetcmd.go b/cmd/devp2p/nodesetcmd.go index 6fbc185ad8..f0773edfb8 100644 --- a/cmd/devp2p/nodesetcmd.go +++ b/cmd/devp2p/nodesetcmd.go @@ -19,7 +19,7 @@ package main import ( "errors" "fmt" - "net" + "net/netip" "sort" "strconv" "strings" @@ -205,11 +205,11 @@ func trueFilter(args []string) (nodeFilter, error) { } func ipFilter(args []string) (nodeFilter, error) { - _, cidr, err := net.ParseCIDR(args[0]) + prefix, err := netip.ParsePrefix(args[0]) if err != nil { return nil, err } - f := func(n nodeJSON) bool { return cidr.Contains(n.N.IP()) } + f := func(n nodeJSON) bool { return prefix.Contains(n.N.IPAddr()) } return f, nil } diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index aa7d065818..fb8066ee1a 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -77,7 +77,11 @@ var ( func rlpxPing(ctx *cli.Context) error { n := getNodeArg(ctx) - fd, err := net.Dial("tcp", fmt.Sprintf("%v:%d", n.IP(), n.TCP())) + tcpEndpoint, ok := n.TCPEndpoint() + if !ok { + return fmt.Errorf("node has no TCP endpoint") + } + fd, err := net.Dial("tcp", tcpEndpoint.String()) if err != nil { return err } diff --git a/p2p/dial.go b/p2p/dial.go index 08e1db2877..24d4dc2e89 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -65,11 +65,8 @@ type tcpDialer struct { } func (t tcpDialer) Dial(ctx context.Context, dest *enode.Node) (net.Conn, error) { - return t.d.DialContext(ctx, "tcp", nodeAddr(dest).String()) -} - -func nodeAddr(n *enode.Node) net.Addr { - return &net.TCPAddr{IP: n.IP(), Port: n.TCP()} + addr, _ := dest.TCPEndpoint() + return t.d.DialContext(ctx, "tcp", addr.String()) } // checkDial errors: @@ -243,7 +240,7 @@ loop: select { case node := <-nodesCh: if err := d.checkDial(node); err != nil { - d.log.Trace("Discarding dial candidate", "id", node.ID(), "ip", node.IP(), "reason", err) + d.log.Trace("Discarding dial candidate", "id", node.ID(), "ip", node.IPAddr(), "reason", err) } else { d.startDial(newDialTask(node, dynDialedConn)) } @@ -277,7 +274,7 @@ loop: case node := <-d.addStaticCh: id := node.ID() _, exists := d.static[id] - d.log.Trace("Adding static node", "id", id, "ip", node.IP(), "added", !exists) + d.log.Trace("Adding static node", "id", id, "ip", node.IPAddr(), "added", !exists) if exists { continue loop } @@ -376,7 +373,7 @@ func (d *dialScheduler) checkDial(n *enode.Node) error { if n.ID() == d.self { return errSelf } - if n.IP() != nil && n.TCP() == 0 { + if n.IPAddr().IsValid() && n.TCP() == 0 { // This check can trigger if a non-TCP node is found // by discovery. If there is no IP, the node is a static // node and the actual endpoint will be resolved later in dialTask. @@ -388,7 +385,7 @@ func (d *dialScheduler) checkDial(n *enode.Node) error { if _, ok := d.peers[n.ID()]; ok { return errAlreadyConnected } - if d.netRestrict != nil && !d.netRestrict.Contains(n.IP()) { + if d.netRestrict != nil && !d.netRestrict.ContainsAddr(n.IPAddr()) { return errNetRestrict } if d.history.contains(string(n.ID().Bytes())) { @@ -439,7 +436,7 @@ func (d *dialScheduler) removeFromStaticPool(idx int) { // startDial runs the given dial task in a separate goroutine. func (d *dialScheduler) startDial(task *dialTask) { node := task.dest() - d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IP(), "flag", task.flags) + d.log.Trace("Starting p2p dial", "id", node.ID(), "ip", node.IPAddr(), "flag", task.flags) hkey := string(node.ID().Bytes()) d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) d.dialing[node.ID()] = task @@ -492,7 +489,7 @@ func (t *dialTask) run(d *dialScheduler) { } func (t *dialTask) needResolve() bool { - return t.flags&staticDialedConn != 0 && t.dest().IP() == nil + return t.flags&staticDialedConn != 0 && !t.dest().IPAddr().IsValid() } // resolve attempts to find the current endpoint for the destination @@ -526,7 +523,8 @@ func (t *dialTask) resolve(d *dialScheduler) bool { // The node was found. t.resolveDelay = initialResolveDelay t.destPtr.Store(resolved) - d.log.Debug("Resolved node", "id", resolved.ID(), "addr", &net.TCPAddr{IP: resolved.IP(), Port: resolved.TCP()}) + resAddr, _ := resolved.TCPEndpoint() + d.log.Debug("Resolved node", "id", resolved.ID(), "addr", resAddr) return true } @@ -535,7 +533,8 @@ func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { dialMeter.Mark(1) fd, err := d.dialer.Dial(d.ctx, dest) if err != nil { - d.log.Trace("Dial error", "id", dest.ID(), "addr", nodeAddr(dest), "conn", t.flags, "err", cleanupDialErr(err)) + addr, _ := dest.TCPEndpoint() + d.log.Trace("Dial error", "id", dest.ID(), "addr", addr, "conn", t.flags, "err", cleanupDialErr(err)) dialConnectionError.Mark(1) return &dialError{err} } @@ -545,7 +544,7 @@ func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { func (t *dialTask) String() string { node := t.dest() id := node.ID() - return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], node.IP(), node.TCP()) + return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], node.IPAddr(), node.TCP()) } func cleanupDialErr(err error) error { diff --git a/p2p/discover/table.go b/p2p/discover/table.go index bd3c9b4143..bb5ab4f3fc 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -25,7 +25,7 @@ package discover import ( "context" "fmt" - "net" + "net/netip" "slices" "sync" "time" @@ -207,8 +207,8 @@ func (tab *Table) setFallbackNodes(nodes []*enode.Node) error { if err := n.ValidateComplete(); err != nil { return fmt.Errorf("bad bootstrap node %q: %v", n, err) } - if tab.cfg.NetRestrict != nil && !tab.cfg.NetRestrict.Contains(n.IP()) { - tab.log.Error("Bootstrap node filtered by netrestrict", "id", n.ID(), "ip", n.IP()) + if tab.cfg.NetRestrict != nil && !tab.cfg.NetRestrict.ContainsAddr(n.IPAddr()) { + tab.log.Error("Bootstrap node filtered by netrestrict", "id", n.ID(), "ip", n.IPAddr()) continue } nursery = append(nursery, n) @@ -448,7 +448,7 @@ func (tab *Table) loadSeedNodes() { for i := range seeds { seed := seeds[i] if tab.log.Enabled(context.Background(), log.LevelTrace) { - age := time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) + age := time.Since(tab.db.LastPongReceived(seed.ID(), seed.IPAddr())) addr, _ := seed.UDPEndpoint() tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", addr, "age", age) } @@ -474,31 +474,31 @@ func (tab *Table) bucketAtDistance(d int) *bucket { return tab.buckets[d-bucketMinDistance-1] } -func (tab *Table) addIP(b *bucket, ip net.IP) bool { - if len(ip) == 0 { +func (tab *Table) addIP(b *bucket, ip netip.Addr) bool { + if !ip.IsValid() || ip.IsUnspecified() { return false // Nodes without IP cannot be added. } - if netutil.IsLAN(ip) { + if netutil.AddrIsLAN(ip) { return true } - if !tab.ips.Add(ip) { + if !tab.ips.AddAddr(ip) { tab.log.Debug("IP exceeds table limit", "ip", ip) return false } - if !b.ips.Add(ip) { + if !b.ips.AddAddr(ip) { tab.log.Debug("IP exceeds bucket limit", "ip", ip) - tab.ips.Remove(ip) + tab.ips.RemoveAddr(ip) return false } return true } -func (tab *Table) removeIP(b *bucket, ip net.IP) { - if netutil.IsLAN(ip) { +func (tab *Table) removeIP(b *bucket, ip netip.Addr) { + if netutil.AddrIsLAN(ip) { return } - tab.ips.Remove(ip) - b.ips.Remove(ip) + tab.ips.RemoveAddr(ip) + b.ips.RemoveAddr(ip) } // handleAddNode adds the node in the request to the table, if there is space. @@ -524,7 +524,7 @@ func (tab *Table) handleAddNode(req addNodeOp) bool { tab.addReplacement(b, req.node) return false } - if !tab.addIP(b, req.node.IP()) { + if !tab.addIP(b, req.node.IPAddr()) { // Can't add: IP limit reached. return false } @@ -547,7 +547,7 @@ func (tab *Table) addReplacement(b *bucket, n *enode.Node) { // TODO: update ENR return } - if !tab.addIP(b, n.IP()) { + if !tab.addIP(b, n.IPAddr()) { return } @@ -555,7 +555,7 @@ func (tab *Table) addReplacement(b *bucket, n *enode.Node) { var removed *tableNode b.replacements, removed = pushNode(b.replacements, wn, maxReplacements) if removed != nil { - tab.removeIP(b, removed.IP()) + tab.removeIP(b, removed.IPAddr()) } } @@ -595,12 +595,12 @@ func (tab *Table) deleteInBucket(b *bucket, id enode.ID) *tableNode { // Remove the node. n := b.entries[index] b.entries = slices.Delete(b.entries, index, index+1) - tab.removeIP(b, n.IP()) + tab.removeIP(b, n.IPAddr()) tab.nodeRemoved(b, n) // Add replacement. if len(b.replacements) == 0 { - tab.log.Debug("Removed dead node", "b", b.index, "id", n.ID(), "ip", n.IP()) + tab.log.Debug("Removed dead node", "b", b.index, "id", n.ID(), "ip", n.IPAddr()) return nil } rindex := tab.rand.Intn(len(b.replacements)) @@ -608,7 +608,7 @@ func (tab *Table) deleteInBucket(b *bucket, id enode.ID) *tableNode { b.replacements = slices.Delete(b.replacements, rindex, rindex+1) b.entries = append(b.entries, rep) tab.nodeAdded(b, rep) - tab.log.Debug("Replaced dead node", "b", b.index, "id", n.ID(), "ip", n.IP(), "r", rep.ID(), "rip", rep.IP()) + tab.log.Debug("Replaced dead node", "b", b.index, "id", n.ID(), "ip", n.IPAddr(), "r", rep.ID(), "rip", rep.IPAddr()) return rep } @@ -635,10 +635,10 @@ func (tab *Table) bumpInBucket(b *bucket, newRecord *enode.Node, isInbound bool) ipchanged := newRecord.IPAddr() != n.IPAddr() portchanged := newRecord.UDP() != n.UDP() if ipchanged { - tab.removeIP(b, n.IP()) - if !tab.addIP(b, newRecord.IP()) { + tab.removeIP(b, n.IPAddr()) + if !tab.addIP(b, newRecord.IPAddr()) { // It doesn't fit with the limit, put the previous record back. - tab.addIP(b, n.IP()) + tab.addIP(b, n.IPAddr()) return n, false } } @@ -657,11 +657,11 @@ func (tab *Table) handleTrackRequest(op trackRequestOp) { var fails int if op.success { // Reset failure counter because it counts _consecutive_ failures. - tab.db.UpdateFindFails(op.node.ID(), op.node.IP(), 0) + tab.db.UpdateFindFails(op.node.ID(), op.node.IPAddr(), 0) } else { - fails = tab.db.FindFails(op.node.ID(), op.node.IP()) + fails = tab.db.FindFails(op.node.ID(), op.node.IPAddr()) fails++ - tab.db.UpdateFindFails(op.node.ID(), op.node.IP(), fails) + tab.db.UpdateFindFails(op.node.ID(), op.node.IPAddr(), fails) } tab.mutex.Lock() diff --git a/p2p/discover/table_test.go b/p2p/discover/table_test.go index 30e7d56f4a..2f1797d1e2 100644 --- a/p2p/discover/table_test.go +++ b/p2p/discover/table_test.go @@ -188,7 +188,7 @@ func checkIPLimitInvariant(t *testing.T, tab *Table) { tabset := netutil.DistinctNetSet{Subnet: tableSubnet, Limit: tableIPLimit} for _, b := range tab.buckets { for _, n := range b.entries { - tabset.Add(n.IP()) + tabset.AddAddr(n.IPAddr()) } } if tabset.String() != tab.ips.String() { @@ -268,7 +268,7 @@ func (*closeTest) Generate(rand *rand.Rand, size int) reflect.Value { } for _, id := range gen([]enode.ID{}, rand).([]enode.ID) { r := new(enr.Record) - r.Set(enr.IP(genIP(rand))) + r.Set(enr.IPv4Addr(netutil.RandomAddr(rand, true))) n := enode.SignNull(r, id) t.All = append(t.All, n) } @@ -385,11 +385,11 @@ func checkBucketContent(t *testing.T, tab *Table, nodes []*enode.Node) { } t.Log("wrong bucket content. have nodes:") for _, n := range b.entries { - t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IP()) + t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IPAddr()) } t.Log("want nodes:") for _, n := range nodes { - t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IP()) + t.Logf(" %v (seq=%v, ip=%v)", n.ID(), n.Seq(), n.IPAddr()) } t.FailNow() @@ -483,12 +483,6 @@ func gen(typ interface{}, rand *rand.Rand) interface{} { return v.Interface() } -func genIP(rand *rand.Rand) net.IP { - ip := make(net.IP, 4) - rand.Read(ip) - return ip -} - func quickcfg() *quick.Config { return &quick.Config{ MaxCount: 5000, diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 997ac37799..5b2699d460 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -100,8 +100,9 @@ func idAtDistance(a enode.ID, n int) (b enode.ID) { return b } +// intIP returns a LAN IP address based on i. func intIP(i int) net.IP { - return net.IP{byte(i), 0, 2, byte(i)} + return net.IP{10, 0, byte(i >> 8), byte(i & 0xFF)} } // fillBucket inserts nodes into the given bucket until it is full. @@ -254,7 +255,7 @@ NotEqual: } func nodeEqual(n1 *enode.Node, n2 *enode.Node) bool { - return n1.ID() == n2.ID() && n1.IP().Equal(n2.IP()) + return n1.ID() == n2.ID() && n1.IPAddr() == n2.IPAddr() } func sortByID[N nodeType](nodes []N) { diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 3880ca34a7..6234fd93ca 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -25,7 +25,6 @@ import ( "errors" "fmt" "io" - "net" "net/netip" "sync" "time" @@ -250,8 +249,7 @@ func (t *UDPv4) sendPing(toid enode.ID, toaddr netip.AddrPort, callback func()) return matched, matched }) // Send the packet. - toUDPAddr := &net.UDPAddr{IP: toaddr.Addr().AsSlice()} - t.localNode.UDPContact(toUDPAddr) + t.localNode.UDPContact(toaddr) t.write(toaddr, toid, req.Name(), packet) return rm } @@ -383,7 +381,7 @@ func (t *UDPv4) RequestENR(n *enode.Node) (*enode.Node, error) { if respN.Seq() < n.Seq() { return n, nil // response record is older } - if err := netutil.CheckRelayIP(addr.Addr().AsSlice(), respN.IP()); err != nil { + if err := netutil.CheckRelayAddr(addr.Addr(), respN.IPAddr()); err != nil { return nil, fmt.Errorf("invalid IP in response record: %v", err) } return respN, nil @@ -578,15 +576,14 @@ func (t *UDPv4) handlePacket(from netip.AddrPort, buf []byte) error { // checkBond checks if the given node has a recent enough endpoint proof. func (t *UDPv4) checkBond(id enode.ID, ip netip.AddrPort) bool { - return time.Since(t.db.LastPongReceived(id, ip.Addr().AsSlice())) < bondExpiration + return time.Since(t.db.LastPongReceived(id, ip.Addr())) < bondExpiration } // ensureBond solicits a ping from a node if we haven't seen a ping from it for a while. // This ensures there is a valid endpoint proof on the remote end. func (t *UDPv4) ensureBond(toid enode.ID, toaddr netip.AddrPort) { - ip := toaddr.Addr().AsSlice() - tooOld := time.Since(t.db.LastPingReceived(toid, ip)) > bondExpiration - if tooOld || t.db.FindFails(toid, ip) > maxFindnodeFailures { + tooOld := time.Since(t.db.LastPingReceived(toid, toaddr.Addr())) > bondExpiration + if tooOld || t.db.FindFails(toid, toaddr.Addr()) > maxFindnodeFailures { rm := t.sendPing(toid, toaddr, nil) <-rm.errc // Wait for them to ping back and process our pong. @@ -687,7 +684,7 @@ func (t *UDPv4) handlePing(h *packetHandlerV4, from netip.AddrPort, fromID enode // Ping back if our last pong on file is too far in the past. fromIP := from.Addr().AsSlice() n := enode.NewV4(h.senderKey, fromIP, int(req.From.TCP), int(from.Port())) - if time.Since(t.db.LastPongReceived(n.ID(), fromIP)) > bondExpiration { + if time.Since(t.db.LastPongReceived(n.ID(), from.Addr())) > bondExpiration { t.sendPing(fromID, from, func() { t.tab.addInboundNode(n) }) @@ -696,10 +693,9 @@ func (t *UDPv4) handlePing(h *packetHandlerV4, from netip.AddrPort, fromID enode } // Update node database and endpoint predictor. - t.db.UpdateLastPingReceived(n.ID(), fromIP, time.Now()) - fromUDPAddr := &net.UDPAddr{IP: fromIP, Port: int(from.Port())} - toUDPAddr := &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)} - t.localNode.UDPEndpointStatement(fromUDPAddr, toUDPAddr) + t.db.UpdateLastPingReceived(n.ID(), from.Addr(), time.Now()) + toaddr := netip.AddrPortFrom(netutil.IPToAddr(req.To.IP), req.To.UDP) + t.localNode.UDPEndpointStatement(from, toaddr) } // PONG/v4 @@ -713,11 +709,9 @@ func (t *UDPv4) verifyPong(h *packetHandlerV4, from netip.AddrPort, fromID enode if !t.handleReply(fromID, from.Addr(), req) { return errUnsolicitedReply } - fromIP := from.Addr().AsSlice() - fromUDPAddr := &net.UDPAddr{IP: fromIP, Port: int(from.Port())} - toUDPAddr := &net.UDPAddr{IP: req.To.IP, Port: int(req.To.UDP)} - t.localNode.UDPEndpointStatement(fromUDPAddr, toUDPAddr) - t.db.UpdateLastPongReceived(fromID, fromIP, time.Now()) + toaddr := netip.AddrPortFrom(netutil.IPToAddr(req.To.IP), req.To.UDP) + t.localNode.UDPEndpointStatement(from, toaddr) + t.db.UpdateLastPongReceived(fromID, from.Addr(), time.Now()) return nil } @@ -753,8 +747,7 @@ func (t *UDPv4) handleFindnode(h *packetHandlerV4, from netip.AddrPort, fromID e p := v4wire.Neighbors{Expiration: uint64(time.Now().Add(expiration).Unix())} var sent bool for _, n := range closest { - fromIP := from.Addr().AsSlice() - if netutil.CheckRelayIP(fromIP, n.IP()) == nil { + if netutil.CheckRelayAddr(from.Addr(), n.IPAddr()) == nil { p.Nodes = append(p.Nodes, nodeToRPC(n)) } if len(p.Nodes) == v4wire.MaxNeighbors { diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 28a6fb8675..9d6df08ead 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -274,7 +274,7 @@ func TestUDPv4_findnode(t *testing.T) { // ensure there's a bond with the test node, // findnode won't be accepted otherwise. remoteID := v4wire.EncodePubkey(&test.remotekey.PublicKey).ID() - test.table.db.UpdateLastPongReceived(remoteID, test.remoteaddr.Addr().AsSlice(), time.Now()) + test.table.db.UpdateLastPongReceived(remoteID, test.remoteaddr.Addr(), time.Now()) // check that closest neighbors are returned. expected := test.table.findnodeByID(testTarget.ID(), bucketSize, true) @@ -309,7 +309,7 @@ func TestUDPv4_findnodeMultiReply(t *testing.T) { defer test.close() rid := enode.PubkeyToIDV4(&test.remotekey.PublicKey) - test.table.db.UpdateLastPingReceived(rid, test.remoteaddr.Addr().AsSlice(), time.Now()) + test.table.db.UpdateLastPingReceived(rid, test.remoteaddr.Addr(), time.Now()) // queue a pending findnode request resultc, errc := make(chan []*enode.Node, 1), make(chan error, 1) @@ -437,8 +437,8 @@ func TestUDPv4_successfulPing(t *testing.T) { if n.ID() != rid { t.Errorf("node has wrong ID: got %v, want %v", n.ID(), rid) } - if !n.IP().Equal(test.remoteaddr.Addr().AsSlice()) { - t.Errorf("node has wrong IP: got %v, want: %v", n.IP(), test.remoteaddr.Addr()) + if n.IPAddr() != test.remoteaddr.Addr() { + t.Errorf("node has wrong IP: got %v, want: %v", n.IPAddr(), test.remoteaddr.Addr()) } if n.UDP() != int(test.remoteaddr.Port()) { t.Errorf("node has wrong UDP port: got %v, want: %v", n.UDP(), test.remoteaddr.Port()) diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 9ba54b3d40..4cc21cd9c5 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -428,10 +428,10 @@ func (t *UDPv5) verifyResponseNode(c *callV5, r *enr.Record, distances []uint, s if err != nil { return nil, err } - if err := netutil.CheckRelayIP(c.addr.Addr().AsSlice(), node.IP()); err != nil { + if err := netutil.CheckRelayAddr(c.addr.Addr(), node.IPAddr()); err != nil { return nil, err } - if t.netrestrict != nil && !t.netrestrict.Contains(node.IP()) { + if t.netrestrict != nil && !t.netrestrict.ContainsAddr(node.IPAddr()) { return nil, errors.New("not contained in netrestrict list") } if node.UDP() <= 1024 { @@ -754,9 +754,8 @@ func (t *UDPv5) handle(p v5wire.Packet, fromID enode.ID, fromAddr netip.AddrPort t.handlePing(p, fromID, fromAddr) case *v5wire.Pong: if t.handleCallResponse(fromID, fromAddr, p) { - fromUDPAddr := &net.UDPAddr{IP: fromAddr.Addr().AsSlice(), Port: int(fromAddr.Port())} - toUDPAddr := &net.UDPAddr{IP: p.ToIP, Port: int(p.ToPort)} - t.localNode.UDPEndpointStatement(fromUDPAddr, toUDPAddr) + toAddr := netip.AddrPortFrom(netutil.IPToAddr(p.ToIP), p.ToPort) + t.localNode.UDPEndpointStatement(fromAddr, toAddr) } case *v5wire.Findnode: t.handleFindnode(p, fromID, fromAddr) @@ -848,7 +847,6 @@ func (t *UDPv5) handleFindnode(p *v5wire.Findnode, fromID enode.ID, fromAddr net // collectTableNodes creates a FINDNODE result set for the given distances. func (t *UDPv5) collectTableNodes(rip netip.Addr, distances []uint, limit int) []*enode.Node { - ripSlice := rip.AsSlice() var bn []*enode.Node var nodes []*enode.Node var processed = make(map[uint]struct{}) @@ -863,7 +861,7 @@ func (t *UDPv5) collectTableNodes(rip netip.Addr, distances []uint, limit int) [ for _, n := range t.tab.appendLiveNodes(dist, bn[:0]) { // Apply some pre-checks to avoid sending invalid nodes. // Note liveness is checked by appendLiveNodes. - if netutil.CheckRelayIP(ripSlice, n.IP()) != nil { + if netutil.CheckRelayAddr(rip, n.IPAddr()) != nil { continue } nodes = append(nodes, n) diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 27966f2afc..8dd02620eb 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -606,7 +606,7 @@ func (n *handshakeTestNode) n() *enode.Node { } func (n *handshakeTestNode) addr() string { - return n.ln.Node().IP().String() + return n.ln.Node().IPAddr().String() } func (n *handshakeTestNode) id() enode.ID { diff --git a/p2p/enode/localnode.go b/p2p/enode/localnode.go index a18204e752..6e79c9cbdc 100644 --- a/p2p/enode/localnode.go +++ b/p2p/enode/localnode.go @@ -20,8 +20,8 @@ import ( "crypto/ecdsa" "fmt" "net" + "net/netip" "reflect" - "strconv" "sync" "sync/atomic" "time" @@ -175,8 +175,8 @@ func (ln *LocalNode) delete(e enr.Entry) { } } -func (ln *LocalNode) endpointForIP(ip net.IP) *lnEndpoint { - if ip.To4() != nil { +func (ln *LocalNode) endpointForIP(ip netip.Addr) *lnEndpoint { + if ip.Is4() { return &ln.endpoint4 } return &ln.endpoint6 @@ -188,7 +188,7 @@ func (ln *LocalNode) SetStaticIP(ip net.IP) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(ip).staticIP = ip + ln.endpointForIP(netutil.IPToAddr(ip)).staticIP = ip ln.updateEndpoints() } @@ -198,7 +198,7 @@ func (ln *LocalNode) SetFallbackIP(ip net.IP) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(ip).fallbackIP = ip + ln.endpointForIP(netutil.IPToAddr(ip)).fallbackIP = ip ln.updateEndpoints() } @@ -215,21 +215,21 @@ func (ln *LocalNode) SetFallbackUDP(port int) { // UDPEndpointStatement should be called whenever a statement about the local node's // UDP endpoint is received. It feeds the local endpoint predictor. -func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint *net.UDPAddr) { +func (ln *LocalNode) UDPEndpointStatement(fromaddr, endpoint netip.AddrPort) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(endpoint.IP).track.AddStatement(fromaddr.String(), endpoint.String()) + ln.endpointForIP(endpoint.Addr()).track.AddStatement(fromaddr.Addr(), endpoint) ln.updateEndpoints() } // UDPContact should be called whenever the local node has announced itself to another node // via UDP. It feeds the local endpoint predictor. -func (ln *LocalNode) UDPContact(toaddr *net.UDPAddr) { +func (ln *LocalNode) UDPContact(toaddr netip.AddrPort) { ln.mu.Lock() defer ln.mu.Unlock() - ln.endpointForIP(toaddr.IP).track.AddContact(toaddr.String()) + ln.endpointForIP(toaddr.Addr()).track.AddContact(toaddr.Addr()) ln.updateEndpoints() } @@ -268,29 +268,13 @@ func (e *lnEndpoint) get() (newIP net.IP, newPort uint16) { } if e.staticIP != nil { newIP = e.staticIP - } else if ip, port := predictAddr(e.track); ip != nil { - newIP = ip - newPort = port + } else if ap := e.track.PredictEndpoint(); ap.IsValid() { + newIP = ap.Addr().AsSlice() + newPort = ap.Port() } return newIP, newPort } -// predictAddr wraps IPTracker.PredictEndpoint, converting from its string-based -// endpoint representation to IP and port types. -func predictAddr(t *netutil.IPTracker) (net.IP, uint16) { - ep := t.PredictEndpoint() - if ep == "" { - return nil, 0 - } - ipString, portString, _ := net.SplitHostPort(ep) - ip := net.ParseIP(ipString) - port, err := strconv.ParseUint(portString, 10, 16) - if err != nil { - return nil, 0 - } - return ip, uint16(port) -} - func (ln *LocalNode) invalidate() { ln.cur.Store((*Node)(nil)) } @@ -314,7 +298,7 @@ func (ln *LocalNode) sign() { panic(fmt.Errorf("enode: can't verify local record: %v", err)) } ln.cur.Store(n) - log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IP(), "udp", n.UDP(), "tcp", n.TCP()) + log.Info("New local node record", "seq", ln.seq, "id", n.ID(), "ip", n.IPAddr(), "udp", n.UDP(), "tcp", n.TCP()) } func (ln *LocalNode) bumpSeq() { diff --git a/p2p/enode/localnode_test.go b/p2p/enode/localnode_test.go index 7f97ad392f..86b962a74e 100644 --- a/p2p/enode/localnode_test.go +++ b/p2p/enode/localnode_test.go @@ -17,12 +17,14 @@ package enode import ( - "crypto/rand" + "math/rand" "net" + "net/netip" "testing" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/stretchr/testify/assert" ) @@ -88,6 +90,7 @@ func TestLocalNodeSeqPersist(t *testing.T) { // This test checks behavior of the endpoint predictor. func TestLocalNodeEndpoint(t *testing.T) { var ( + rng = rand.New(rand.NewSource(4)) fallback = &net.UDPAddr{IP: net.IP{127, 0, 0, 1}, Port: 80} predicted = &net.UDPAddr{IP: net.IP{127, 0, 1, 2}, Port: 81} staticIP = net.IP{127, 0, 1, 2} @@ -96,6 +99,7 @@ func TestLocalNodeEndpoint(t *testing.T) { defer db.Close() // Nothing is set initially. + assert.Equal(t, netip.Addr{}, ln.Node().IPAddr()) assert.Equal(t, net.IP(nil), ln.Node().IP()) assert.Equal(t, 0, ln.Node().UDP()) initialSeq := ln.Node().Seq() @@ -103,26 +107,30 @@ func TestLocalNodeEndpoint(t *testing.T) { // Set up fallback address. ln.SetFallbackIP(fallback.IP) ln.SetFallbackUDP(fallback.Port) + assert.Equal(t, netutil.IPToAddr(fallback.IP), ln.Node().IPAddr()) assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) assert.Equal(t, initialSeq+1, ln.Node().Seq()) // Add endpoint statements from random hosts. for i := 0; i < iptrackMinStatements; i++ { + assert.Equal(t, netutil.IPToAddr(fallback.IP), ln.Node().IPAddr()) assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) assert.Equal(t, initialSeq+1, ln.Node().Seq()) - from := &net.UDPAddr{IP: make(net.IP, 4), Port: 90} - rand.Read(from.IP) - ln.UDPEndpointStatement(from, predicted) + from := netip.AddrPortFrom(netutil.RandomAddr(rng, true), 9000) + endpoint := netip.AddrPortFrom(netutil.IPToAddr(predicted.IP), uint16(predicted.Port)) + ln.UDPEndpointStatement(from, endpoint) } + assert.Equal(t, netutil.IPToAddr(predicted.IP), ln.Node().IPAddr()) assert.Equal(t, predicted.IP, ln.Node().IP()) assert.Equal(t, predicted.Port, ln.Node().UDP()) assert.Equal(t, initialSeq+2, ln.Node().Seq()) // Static IP overrides prediction. ln.SetStaticIP(staticIP) + assert.Equal(t, netutil.IPToAddr(staticIP), ln.Node().IPAddr()) assert.Equal(t, staticIP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) assert.Equal(t, initialSeq+3, ln.Node().Seq()) diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index 654d71d47b..1f31c98d22 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -21,7 +21,7 @@ import ( "crypto/rand" "encoding/binary" "fmt" - "net" + "net/netip" "os" "sync" "time" @@ -66,7 +66,7 @@ var ( errInvalidIP = errors.New("invalid IP") ) -var zeroIP = make(net.IP, 16) +var zeroIP = netip.IPv6Unspecified() // DB is the node database, storing previously seen nodes and any collected metadata about // them for QoS purposes. @@ -151,39 +151,37 @@ func splitNodeKey(key []byte) (id ID, rest []byte) { } // nodeItemKey returns the database key for a node metadata field. -func nodeItemKey(id ID, ip net.IP, field string) []byte { - ip16 := ip.To16() - if ip16 == nil { - panic(fmt.Errorf("invalid IP (length %d)", len(ip))) +func nodeItemKey(id ID, ip netip.Addr, field string) []byte { + if !ip.IsValid() { + panic("invalid IP") } - return bytes.Join([][]byte{nodeKey(id), ip16, []byte(field)}, []byte{':'}) + ip16 := ip.As16() + return bytes.Join([][]byte{nodeKey(id), ip16[:], []byte(field)}, []byte{':'}) } // splitNodeItemKey returns the components of a key created by nodeItemKey. -func splitNodeItemKey(key []byte) (id ID, ip net.IP, field string) { +func splitNodeItemKey(key []byte) (id ID, ip netip.Addr, field string) { id, key = splitNodeKey(key) // Skip discover root. if string(key) == dbDiscoverRoot { - return id, nil, "" + return id, netip.Addr{}, "" } key = key[len(dbDiscoverRoot)+1:] // Split out the IP. - ip = key[:16] - if ip4 := ip.To4(); ip4 != nil { - ip = ip4 - } + ip, _ = netip.AddrFromSlice(key[:16]) key = key[16+1:] // Field is the remainder of key. field = string(key) return id, ip, field } -func v5Key(id ID, ip net.IP, field string) []byte { +func v5Key(id ID, ip netip.Addr, field string) []byte { + ip16 := ip.As16() return bytes.Join([][]byte{ []byte(dbNodePrefix), id[:], []byte(dbDiscv5Root), - ip.To16(), + ip16[:], []byte(field), }, []byte{':'}) } @@ -364,24 +362,24 @@ func (db *DB) expireNodes() { // LastPingReceived retrieves the time of the last ping packet received from // a remote node. -func (db *DB) LastPingReceived(id ID, ip net.IP) time.Time { - if ip = ip.To16(); ip == nil { +func (db *DB) LastPingReceived(id ID, ip netip.Addr) time.Time { + if !ip.IsValid() { return time.Time{} } return time.Unix(db.fetchInt64(nodeItemKey(id, ip, dbNodePing)), 0) } // UpdateLastPingReceived updates the last time we tried contacting a remote node. -func (db *DB) UpdateLastPingReceived(id ID, ip net.IP, instance time.Time) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateLastPingReceived(id ID, ip netip.Addr, instance time.Time) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodePing), instance.Unix()) } // LastPongReceived retrieves the time of the last successful pong from remote node. -func (db *DB) LastPongReceived(id ID, ip net.IP) time.Time { - if ip = ip.To16(); ip == nil { +func (db *DB) LastPongReceived(id ID, ip netip.Addr) time.Time { + if !ip.IsValid() { return time.Time{} } // Launch expirer @@ -390,40 +388,40 @@ func (db *DB) LastPongReceived(id ID, ip net.IP) time.Time { } // UpdateLastPongReceived updates the last pong time of a node. -func (db *DB) UpdateLastPongReceived(id ID, ip net.IP, instance time.Time) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateLastPongReceived(id ID, ip netip.Addr, instance time.Time) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodePong), instance.Unix()) } // FindFails retrieves the number of findnode failures since bonding. -func (db *DB) FindFails(id ID, ip net.IP) int { - if ip = ip.To16(); ip == nil { +func (db *DB) FindFails(id ID, ip netip.Addr) int { + if !ip.IsValid() { return 0 } return int(db.fetchInt64(nodeItemKey(id, ip, dbNodeFindFails))) } // UpdateFindFails updates the number of findnode failures since bonding. -func (db *DB) UpdateFindFails(id ID, ip net.IP, fails int) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateFindFails(id ID, ip netip.Addr, fails int) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(nodeItemKey(id, ip, dbNodeFindFails), int64(fails)) } // FindFailsV5 retrieves the discv5 findnode failure counter. -func (db *DB) FindFailsV5(id ID, ip net.IP) int { - if ip = ip.To16(); ip == nil { +func (db *DB) FindFailsV5(id ID, ip netip.Addr) int { + if !ip.IsValid() { return 0 } return int(db.fetchInt64(v5Key(id, ip, dbNodeFindFails))) } // UpdateFindFailsV5 stores the discv5 findnode failure counter. -func (db *DB) UpdateFindFailsV5(id ID, ip net.IP, fails int) error { - if ip = ip.To16(); ip == nil { +func (db *DB) UpdateFindFailsV5(id ID, ip netip.Addr, fails int) error { + if !ip.IsValid() { return errInvalidIP } return db.storeInt64(v5Key(id, ip, dbNodeFindFails), int64(fails)) @@ -470,7 +468,7 @@ seek: id[0] = 0 continue seek // iterator exhausted } - if now.Sub(db.LastPongReceived(n.ID(), n.IP())) > maxAge { + if now.Sub(db.LastPongReceived(n.ID(), n.IPAddr())) > maxAge { continue seek } for i := range nodes { diff --git a/p2p/enode/nodedb_test.go b/p2p/enode/nodedb_test.go index 38764f31b1..bc0291665d 100644 --- a/p2p/enode/nodedb_test.go +++ b/p2p/enode/nodedb_test.go @@ -20,6 +20,7 @@ import ( "bytes" "fmt" "net" + "net/netip" "path/filepath" "reflect" "testing" @@ -48,8 +49,10 @@ func TestDBNodeKey(t *testing.T) { } func TestDBNodeItemKey(t *testing.T) { - wantIP := net.IP{127, 0, 0, 3} + wantIP := netip.MustParseAddr("127.0.0.3") + wantIP4in6 := netip.AddrFrom16(wantIP.As16()) wantField := "foobar" + enc := nodeItemKey(keytestID, wantIP, wantField) want := []byte{ 'n', ':', @@ -69,7 +72,7 @@ func TestDBNodeItemKey(t *testing.T) { if id != keytestID { t.Errorf("splitNodeItemKey returned wrong ID: %v", id) } - if !ip.Equal(wantIP) { + if ip != wantIP4in6 { t.Errorf("splitNodeItemKey returned wrong IP: %v", ip) } if field != wantField { @@ -123,33 +126,33 @@ func TestDBFetchStore(t *testing.T) { defer db.Close() // Check fetch/store operations on a node ping object - if stored := db.LastPingReceived(node.ID(), node.IP()); stored.Unix() != 0 { + if stored := db.LastPingReceived(node.ID(), node.IPAddr()); stored.Unix() != 0 { t.Errorf("ping: non-existing object: %v", stored) } - if err := db.UpdateLastPingReceived(node.ID(), node.IP(), inst); err != nil { + if err := db.UpdateLastPingReceived(node.ID(), node.IPAddr(), inst); err != nil { t.Errorf("ping: failed to update: %v", err) } - if stored := db.LastPingReceived(node.ID(), node.IP()); stored.Unix() != inst.Unix() { + if stored := db.LastPingReceived(node.ID(), node.IPAddr()); stored.Unix() != inst.Unix() { t.Errorf("ping: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node pong object - if stored := db.LastPongReceived(node.ID(), node.IP()); stored.Unix() != 0 { + if stored := db.LastPongReceived(node.ID(), node.IPAddr()); stored.Unix() != 0 { t.Errorf("pong: non-existing object: %v", stored) } - if err := db.UpdateLastPongReceived(node.ID(), node.IP(), inst); err != nil { + if err := db.UpdateLastPongReceived(node.ID(), node.IPAddr(), inst); err != nil { t.Errorf("pong: failed to update: %v", err) } - if stored := db.LastPongReceived(node.ID(), node.IP()); stored.Unix() != inst.Unix() { + if stored := db.LastPongReceived(node.ID(), node.IPAddr()); stored.Unix() != inst.Unix() { t.Errorf("pong: value mismatch: have %v, want %v", stored, inst) } // Check fetch/store operations on a node findnode-failure object - if stored := db.FindFails(node.ID(), node.IP()); stored != 0 { + if stored := db.FindFails(node.ID(), node.IPAddr()); stored != 0 { t.Errorf("find-node fails: non-existing object: %v", stored) } - if err := db.UpdateFindFails(node.ID(), node.IP(), num); err != nil { + if err := db.UpdateFindFails(node.ID(), node.IPAddr(), num); err != nil { t.Errorf("find-node fails: failed to update: %v", err) } - if stored := db.FindFails(node.ID(), node.IP()); stored != num { + if stored := db.FindFails(node.ID(), node.IPAddr()); stored != num { t.Errorf("find-node fails: value mismatch: have %v, want %v", stored, num) } // Check fetch/store operations on an actual node object @@ -266,7 +269,7 @@ func testSeedQuery() error { if err := db.UpdateNode(seed.node); err != nil { return fmt.Errorf("node %d: failed to insert: %v", i, err) } - if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IP(), seed.pong); err != nil { + if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IPAddr(), seed.pong); err != nil { return fmt.Errorf("node %d: failed to insert bondTime: %v", i, err) } } @@ -427,7 +430,7 @@ func TestDBExpiration(t *testing.T) { t.Fatalf("node %d: failed to insert: %v", i, err) } } - if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IP(), seed.pong); err != nil { + if err := db.UpdateLastPongReceived(seed.node.ID(), seed.node.IPAddr(), seed.pong); err != nil { t.Fatalf("node %d: failed to update bondTime: %v", i, err) } } @@ -438,13 +441,13 @@ func TestDBExpiration(t *testing.T) { unixZeroTime := time.Unix(0, 0) for i, seed := range nodeDBExpirationNodes { node := db.Node(seed.node.ID()) - pong := db.LastPongReceived(seed.node.ID(), seed.node.IP()) + pong := db.LastPongReceived(seed.node.ID(), seed.node.IPAddr()) if seed.exp { if seed.storeNode && node != nil { t.Errorf("node %d (%s) shouldn't be present after expiration", i, seed.node.ID().TerminalString()) } if !pong.Equal(unixZeroTime) { - t.Errorf("pong time %d (%s %v) shouldn't be present after expiration", i, seed.node.ID().TerminalString(), seed.node.IP()) + t.Errorf("pong time %d (%s %v) shouldn't be present after expiration", i, seed.node.ID().TerminalString(), seed.node.IPAddr()) } } else { if seed.storeNode && node == nil { @@ -463,7 +466,7 @@ func TestDBExpireV5(t *testing.T) { db, _ := OpenDB("") defer db.Close() - ip := net.IP{127, 0, 0, 1} + ip := netip.MustParseAddr("127.0.0.1") db.UpdateFindFailsV5(ID{}, ip, 4) db.expireNodes() } diff --git a/p2p/netutil/addrutil.go b/p2p/netutil/addrutil.go index fb6d8d2731..b8b318571b 100644 --- a/p2p/netutil/addrutil.go +++ b/p2p/netutil/addrutil.go @@ -16,18 +16,53 @@ package netutil -import "net" +import ( + "fmt" + "math/rand" + "net" + "net/netip" +) -// AddrIP gets the IP address contained in addr. It returns nil if no address is present. -func AddrIP(addr net.Addr) net.IP { +// AddrAddr gets the IP address contained in addr. The result will be invalid if the +// address type is unsupported. +func AddrAddr(addr net.Addr) netip.Addr { switch a := addr.(type) { case *net.IPAddr: - return a.IP + return IPToAddr(a.IP) case *net.TCPAddr: - return a.IP + return IPToAddr(a.IP) case *net.UDPAddr: - return a.IP + return IPToAddr(a.IP) default: - return nil + return netip.Addr{} } } + +// IPToAddr converts net.IP to netip.Addr. Note that unlike netip.AddrFromSlice, this +// function will always ensure that the resulting Addr is IPv4 when the input is. +func IPToAddr(ip net.IP) netip.Addr { + if ip4 := ip.To4(); ip4 != nil { + addr, _ := netip.AddrFromSlice(ip4) + return addr + } else if ip6 := ip.To16(); ip6 != nil { + addr, _ := netip.AddrFromSlice(ip6) + return addr + } + return netip.Addr{} +} + +// RandomAddr creates a random IP address. +func RandomAddr(rng *rand.Rand, ipv4 bool) netip.Addr { + var bytes []byte + if ipv4 || rng.Intn(2) == 0 { + bytes = make([]byte, 4) + } else { + bytes = make([]byte, 16) + } + rng.Read(bytes) + addr, ok := netip.AddrFromSlice(bytes) + if !ok { + panic(fmt.Errorf("BUG! invalid IP %v", bytes)) + } + return addr +} diff --git a/p2p/netutil/iptrack.go b/p2p/netutil/iptrack.go index a070499e19..5140ac7539 100644 --- a/p2p/netutil/iptrack.go +++ b/p2p/netutil/iptrack.go @@ -17,6 +17,7 @@ package netutil import ( + "net/netip" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -29,14 +30,14 @@ type IPTracker struct { contactWindow time.Duration minStatements int clock mclock.Clock - statements map[string]ipStatement - contact map[string]mclock.AbsTime + statements map[netip.Addr]ipStatement + contact map[netip.Addr]mclock.AbsTime lastStatementGC mclock.AbsTime lastContactGC mclock.AbsTime } type ipStatement struct { - endpoint string + endpoint netip.AddrPort time mclock.AbsTime } @@ -51,9 +52,9 @@ func NewIPTracker(window, contactWindow time.Duration, minStatements int) *IPTra return &IPTracker{ window: window, contactWindow: contactWindow, - statements: make(map[string]ipStatement), + statements: make(map[netip.Addr]ipStatement), minStatements: minStatements, - contact: make(map[string]mclock.AbsTime), + contact: make(map[netip.Addr]mclock.AbsTime), clock: mclock.System{}, } } @@ -74,12 +75,15 @@ func (it *IPTracker) PredictFullConeNAT() bool { } // PredictEndpoint returns the current prediction of the external endpoint. -func (it *IPTracker) PredictEndpoint() string { +func (it *IPTracker) PredictEndpoint() netip.AddrPort { it.gcStatements(it.clock.Now()) // The current strategy is simple: find the endpoint with most statements. - counts := make(map[string]int, len(it.statements)) - maxcount, max := 0, "" + var ( + counts = make(map[netip.AddrPort]int, len(it.statements)) + maxcount int + max netip.AddrPort + ) for _, s := range it.statements { c := counts[s.endpoint] + 1 counts[s.endpoint] = c @@ -91,7 +95,7 @@ func (it *IPTracker) PredictEndpoint() string { } // AddStatement records that a certain host thinks our external endpoint is the one given. -func (it *IPTracker) AddStatement(host, endpoint string) { +func (it *IPTracker) AddStatement(host netip.Addr, endpoint netip.AddrPort) { now := it.clock.Now() it.statements[host] = ipStatement{endpoint, now} if time.Duration(now-it.lastStatementGC) >= it.window { @@ -101,7 +105,7 @@ func (it *IPTracker) AddStatement(host, endpoint string) { // AddContact records that a packet containing our endpoint information has been sent to a // certain host. -func (it *IPTracker) AddContact(host string) { +func (it *IPTracker) AddContact(host netip.Addr) { now := it.clock.Now() it.contact[host] = now if time.Duration(now-it.lastContactGC) >= it.contactWindow { diff --git a/p2p/netutil/iptrack_test.go b/p2p/netutil/iptrack_test.go index ee3bba861e..81653a2733 100644 --- a/p2p/netutil/iptrack_test.go +++ b/p2p/netutil/iptrack_test.go @@ -19,6 +19,7 @@ package netutil import ( crand "crypto/rand" "fmt" + "net/netip" "testing" "time" @@ -42,37 +43,37 @@ func TestIPTracker(t *testing.T) { tests := map[string][]iptrackTestEvent{ "minStatements": { {opPredict, 0, "", ""}, - {opStatement, 0, "127.0.0.1", "127.0.0.2"}, + {opStatement, 0, "127.0.0.1:8000", "127.0.0.2"}, {opPredict, 1000, "", ""}, - {opStatement, 1000, "127.0.0.1", "127.0.0.3"}, + {opStatement, 1000, "127.0.0.1:8000", "127.0.0.3"}, {opPredict, 1000, "", ""}, - {opStatement, 1000, "127.0.0.1", "127.0.0.4"}, - {opPredict, 1000, "127.0.0.1", ""}, + {opStatement, 1000, "127.0.0.1:8000", "127.0.0.4"}, + {opPredict, 1000, "127.0.0.1:8000", ""}, }, "window": { - {opStatement, 0, "127.0.0.1", "127.0.0.2"}, - {opStatement, 2000, "127.0.0.1", "127.0.0.3"}, - {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, - {opPredict, 10000, "127.0.0.1", ""}, + {opStatement, 0, "127.0.0.1:8000", "127.0.0.2"}, + {opStatement, 2000, "127.0.0.1:8000", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1:8000", "127.0.0.4"}, + {opPredict, 10000, "127.0.0.1:8000", ""}, {opPredict, 10001, "", ""}, // first statement expired - {opStatement, 10100, "127.0.0.1", "127.0.0.2"}, - {opPredict, 10200, "127.0.0.1", ""}, + {opStatement, 10100, "127.0.0.1:8000", "127.0.0.2"}, + {opPredict, 10200, "127.0.0.1:8000", ""}, }, "fullcone": { {opContact, 0, "", "127.0.0.2"}, - {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1:8000", "127.0.0.2"}, {opContact, 2000, "", "127.0.0.3"}, - {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, + {opStatement, 2010, "127.0.0.1:8000", "127.0.0.3"}, {opContact, 3000, "", "127.0.0.4"}, - {opStatement, 3010, "127.0.0.1", "127.0.0.4"}, + {opStatement, 3010, "127.0.0.1:8000", "127.0.0.4"}, {opCheckFullCone, 3500, "false", ""}, }, "fullcone_2": { {opContact, 0, "", "127.0.0.2"}, - {opStatement, 10, "127.0.0.1", "127.0.0.2"}, + {opStatement, 10, "127.0.0.1:8000", "127.0.0.2"}, {opContact, 2000, "", "127.0.0.3"}, - {opStatement, 2010, "127.0.0.1", "127.0.0.3"}, - {opStatement, 3000, "127.0.0.1", "127.0.0.4"}, + {opStatement, 2010, "127.0.0.1:8000", "127.0.0.3"}, + {opStatement, 3000, "127.0.0.1:8000", "127.0.0.4"}, {opContact, 3010, "", "127.0.0.4"}, {opCheckFullCone, 3500, "true", ""}, }, @@ -93,12 +94,19 @@ func runIPTrackerTest(t *testing.T, evs []iptrackTestEvent) { clock.Run(evtime - time.Duration(clock.Now())) switch ev.op { case opStatement: - it.AddStatement(ev.from, ev.ip) + it.AddStatement(netip.MustParseAddr(ev.from), netip.MustParseAddrPort(ev.ip)) case opContact: - it.AddContact(ev.from) + it.AddContact(netip.MustParseAddr(ev.from)) case opPredict: - if pred := it.PredictEndpoint(); pred != ev.ip { - t.Errorf("op %d: wrong prediction %q, want %q", i, pred, ev.ip) + pred := it.PredictEndpoint() + if ev.ip == "" { + if pred.IsValid() { + t.Errorf("op %d: wrong prediction %v, expected invalid", i, pred) + } + } else { + if pred != netip.MustParseAddrPort(ev.ip) { + t.Errorf("op %d: wrong prediction %v, want %q", i, pred, ev.ip) + } } case opCheckFullCone: pred := fmt.Sprintf("%t", it.PredictFullConeNAT()) @@ -121,12 +129,11 @@ func TestIPTrackerForceGC(t *testing.T) { it.clock = &clock for i := 0; i < 5*max; i++ { - e1 := make([]byte, 4) - e2 := make([]byte, 4) - crand.Read(e1) - crand.Read(e2) - it.AddStatement(string(e1), string(e2)) - it.AddContact(string(e1)) + var e1, e2 [4]byte + crand.Read(e1[:]) + crand.Read(e2[:]) + it.AddStatement(netip.AddrFrom4(e1), netip.AddrPortFrom(netip.AddrFrom4(e2), 9000)) + it.AddContact(netip.AddrFrom4(e1)) clock.Run(rate) } if len(it.contact) > 2*max { diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index d5da3c694f..474912978e 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -22,21 +22,19 @@ import ( "errors" "fmt" "net" - "sort" + "net/netip" + "slices" "strings" + + "golang.org/x/exp/maps" ) -var lan4, lan6, special4, special6 Netlist +var special4, special6 Netlist func init() { // Lists from RFC 5735, RFC 5156, // https://www.iana.org/assignments/iana-ipv4-special-registry/ - lan4.Add("0.0.0.0/8") // "This" network - lan4.Add("10.0.0.0/8") // Private Use - lan4.Add("172.16.0.0/12") // Private Use - lan4.Add("192.168.0.0/16") // Private Use - lan6.Add("fe80::/10") // Link-Local - lan6.Add("fc00::/7") // Unique-Local + special4.Add("0.0.0.0/8") // "This" network. special4.Add("192.0.0.0/29") // IPv4 Service Continuity special4.Add("192.0.0.9/32") // PCP Anycast special4.Add("192.0.0.170/32") // NAT64/DNS64 Discovery @@ -66,7 +64,7 @@ func init() { } // Netlist is a list of IP networks. -type Netlist []net.IPNet +type Netlist []netip.Prefix // ParseNetlist parses a comma-separated list of CIDR masks. // Whitespace and extra commas are ignored. @@ -78,11 +76,11 @@ func ParseNetlist(s string) (*Netlist, error) { if mask == "" { continue } - _, n, err := net.ParseCIDR(mask) + prefix, err := netip.ParsePrefix(mask) if err != nil { return nil, err } - l = append(l, *n) + l = append(l, prefix) } return &l, nil } @@ -103,11 +101,11 @@ func (l *Netlist) UnmarshalTOML(fn func(interface{}) error) error { return err } for _, mask := range masks { - _, n, err := net.ParseCIDR(mask) + prefix, err := netip.ParsePrefix(mask) if err != nil { return err } - *l = append(*l, *n) + *l = append(*l, prefix) } return nil } @@ -115,15 +113,20 @@ func (l *Netlist) UnmarshalTOML(fn func(interface{}) error) error { // Add parses a CIDR mask and appends it to the list. It panics for invalid masks and is // intended to be used for setting up static lists. func (l *Netlist) Add(cidr string) { - _, n, err := net.ParseCIDR(cidr) + prefix, err := netip.ParsePrefix(cidr) if err != nil { panic(err) } - *l = append(*l, *n) + *l = append(*l, prefix) } // Contains reports whether the given IP is contained in the list. func (l *Netlist) Contains(ip net.IP) bool { + return l.ContainsAddr(IPToAddr(ip)) +} + +// ContainsAddr reports whether the given IP is contained in the list. +func (l *Netlist) ContainsAddr(ip netip.Addr) bool { if l == nil { return false } @@ -137,25 +140,39 @@ func (l *Netlist) Contains(ip net.IP) bool { // IsLAN reports whether an IP is a local network address. func IsLAN(ip net.IP) bool { + return AddrIsLAN(IPToAddr(ip)) +} + +// AddrIsLAN reports whether an IP is a local network address. +func AddrIsLAN(ip netip.Addr) bool { + if ip.Is4In6() { + ip = netip.AddrFrom4(ip.As4()) + } if ip.IsLoopback() { return true } - if v4 := ip.To4(); v4 != nil { - return lan4.Contains(v4) - } - return lan6.Contains(ip) + return ip.IsPrivate() || ip.IsLinkLocalUnicast() } // IsSpecialNetwork reports whether an IP is located in a special-use network range // This includes broadcast, multicast and documentation addresses. func IsSpecialNetwork(ip net.IP) bool { + return AddrIsSpecialNetwork(IPToAddr(ip)) +} + +// AddrIsSpecialNetwork reports whether an IP is located in a special-use network range +// This includes broadcast, multicast and documentation addresses. +func AddrIsSpecialNetwork(ip netip.Addr) bool { + if ip.Is4In6() { + ip = netip.AddrFrom4(ip.As4()) + } if ip.IsMulticast() { return true } - if v4 := ip.To4(); v4 != nil { - return special4.Contains(v4) + if ip.Is4() { + return special4.ContainsAddr(ip) } - return special6.Contains(ip) + return special6.ContainsAddr(ip) } var ( @@ -175,19 +192,31 @@ var ( // - LAN addresses are OK if relayed by a LAN host. // - All other addresses are always acceptable. func CheckRelayIP(sender, addr net.IP) error { - if len(addr) != net.IPv4len && len(addr) != net.IPv6len { + return CheckRelayAddr(IPToAddr(sender), IPToAddr(addr)) +} + +// CheckRelayAddr reports whether an IP relayed from the given sender IP +// is a valid connection target. +// +// There are four rules: +// - Special network addresses are never valid. +// - Loopback addresses are OK if relayed by a loopback host. +// - LAN addresses are OK if relayed by a LAN host. +// - All other addresses are always acceptable. +func CheckRelayAddr(sender, addr netip.Addr) error { + if !addr.IsValid() { return errInvalid } if addr.IsUnspecified() { return errUnspecified } - if IsSpecialNetwork(addr) { + if AddrIsSpecialNetwork(addr) { return errSpecial } if addr.IsLoopback() && !sender.IsLoopback() { return errLoopback } - if IsLAN(addr) && !IsLAN(sender) { + if AddrIsLAN(addr) && !AddrIsLAN(sender) { return errLAN } return nil @@ -221,17 +250,22 @@ type DistinctNetSet struct { Subnet uint // number of common prefix bits Limit uint // maximum number of IPs in each subnet - members map[string]uint - buf net.IP + members map[netip.Prefix]uint } // Add adds an IP address to the set. It returns false (and doesn't add the IP) if the // number of existing IPs in the defined range exceeds the limit. func (s *DistinctNetSet) Add(ip net.IP) bool { + return s.AddAddr(IPToAddr(ip)) +} + +// AddAddr adds an IP address to the set. It returns false (and doesn't add the IP) if the +// number of existing IPs in the defined range exceeds the limit. +func (s *DistinctNetSet) AddAddr(ip netip.Addr) bool { key := s.key(ip) - n := s.members[string(key)] + n := s.members[key] if n < s.Limit { - s.members[string(key)] = n + 1 + s.members[key] = n + 1 return true } return false @@ -239,20 +273,30 @@ func (s *DistinctNetSet) Add(ip net.IP) bool { // Remove removes an IP from the set. func (s *DistinctNetSet) Remove(ip net.IP) { + s.RemoveAddr(IPToAddr(ip)) +} + +// RemoveAddr removes an IP from the set. +func (s *DistinctNetSet) RemoveAddr(ip netip.Addr) { key := s.key(ip) - if n, ok := s.members[string(key)]; ok { + if n, ok := s.members[key]; ok { if n == 1 { - delete(s.members, string(key)) + delete(s.members, key) } else { - s.members[string(key)] = n - 1 + s.members[key] = n - 1 } } } // Contains whether the given IP is contained in the set. func (s DistinctNetSet) Contains(ip net.IP) bool { + return s.ContainsAddr(IPToAddr(ip)) +} + +// ContainsAddr whether the given IP is contained in the set. +func (s DistinctNetSet) ContainsAddr(ip netip.Addr) bool { key := s.key(ip) - _, ok := s.members[string(key)] + _, ok := s.members[key] return ok } @@ -265,54 +309,30 @@ func (s DistinctNetSet) Len() int { return int(n) } -// key encodes the map key for an address into a temporary buffer. -// -// The first byte of key is '4' or '6' to distinguish IPv4/IPv6 address types. -// The remainder of the key is the IP, truncated to the number of bits. -func (s *DistinctNetSet) key(ip net.IP) net.IP { +// key returns the map key for ip. +func (s *DistinctNetSet) key(ip netip.Addr) netip.Prefix { // Lazily initialize storage. if s.members == nil { - s.members = make(map[string]uint) - s.buf = make(net.IP, 17) + s.members = make(map[netip.Prefix]uint) } - // Canonicalize ip and bits. - typ := byte('6') - if ip4 := ip.To4(); ip4 != nil { - typ, ip = '4', ip4 + p, err := ip.Prefix(int(s.Subnet)) + if err != nil { + panic(err) } - bits := s.Subnet - if bits > uint(len(ip)*8) { - bits = uint(len(ip) * 8) - } - // Encode the prefix into s.buf. - nb := int(bits / 8) - mask := ^byte(0xFF >> (bits % 8)) - s.buf[0] = typ - buf := append(s.buf[:1], ip[:nb]...) - if nb < len(ip) && mask != 0 { - buf = append(buf, ip[nb]&mask) - } - return buf + return p } // String implements fmt.Stringer func (s DistinctNetSet) String() string { + keys := maps.Keys(s.members) + slices.SortFunc(keys, func(a, b netip.Prefix) int { + return strings.Compare(a.String(), b.String()) + }) + var buf bytes.Buffer buf.WriteString("{") - keys := make([]string, 0, len(s.members)) - for k := range s.members { - keys = append(keys, k) - } - sort.Strings(keys) for i, k := range keys { - var ip net.IP - if k[0] == '4' { - ip = make(net.IP, 4) - } else { - ip = make(net.IP, 16) - } - copy(ip, k[1:]) - fmt.Fprintf(&buf, "%v×%d", ip, s.members[k]) + fmt.Fprintf(&buf, "%v×%d", k, s.members[k]) if i != len(keys)-1 { buf.WriteString(" ") } diff --git a/p2p/netutil/net_test.go b/p2p/netutil/net_test.go index 3a6aa081f2..569c7ac454 100644 --- a/p2p/netutil/net_test.go +++ b/p2p/netutil/net_test.go @@ -18,7 +18,9 @@ package netutil import ( "fmt" + "math/rand" "net" + "net/netip" "reflect" "testing" "testing/quick" @@ -29,7 +31,7 @@ import ( func TestParseNetlist(t *testing.T) { var tests = []struct { input string - wantErr error + wantErr string wantList *Netlist }{ { @@ -38,25 +40,27 @@ func TestParseNetlist(t *testing.T) { }, { input: "127.0.0.0/8", - wantErr: nil, - wantList: &Netlist{{IP: net.IP{127, 0, 0, 0}, Mask: net.CIDRMask(8, 32)}}, + wantList: &Netlist{netip.MustParsePrefix("127.0.0.0/8")}, }, { input: "127.0.0.0/44", - wantErr: &net.ParseError{Type: "CIDR address", Text: "127.0.0.0/44"}, + wantErr: `netip.ParsePrefix("127.0.0.0/44"): prefix length out of range`, }, { input: "127.0.0.0/16, 23.23.23.23/24,", wantList: &Netlist{ - {IP: net.IP{127, 0, 0, 0}, Mask: net.CIDRMask(16, 32)}, - {IP: net.IP{23, 23, 23, 0}, Mask: net.CIDRMask(24, 32)}, + netip.MustParsePrefix("127.0.0.0/16"), + netip.MustParsePrefix("23.23.23.23/24"), }, }, } for _, test := range tests { l, err := ParseNetlist(test.input) - if !reflect.DeepEqual(err, test.wantErr) { + if err == nil && test.wantErr != "" { + t.Errorf("%q: got no error, expected %q", test.input, test.wantErr) + continue + } else if err != nil && err.Error() != test.wantErr { t.Errorf("%q: got error %q, want %q", test.input, err, test.wantErr) continue } @@ -70,14 +74,12 @@ func TestParseNetlist(t *testing.T) { func TestNilNetListContains(t *testing.T) { var list *Netlist - checkContains(t, list.Contains, nil, []string{"1.2.3.4"}) + checkContains(t, list.Contains, list.ContainsAddr, nil, []string{"1.2.3.4"}) } func TestIsLAN(t *testing.T) { - checkContains(t, IsLAN, + checkContains(t, IsLAN, AddrIsLAN, []string{ // included - "0.0.0.0", - "0.2.0.8", "127.0.0.1", "10.0.1.1", "10.22.0.3", @@ -86,25 +88,35 @@ func TestIsLAN(t *testing.T) { "fe80::f4a1:8eff:fec5:9d9d", "febf::ab32:2233", "fc00::4", + // 4-in-6 + "::ffff:127.0.0.1", + "::ffff:10.10.0.2", }, []string{ // excluded "192.0.2.1", "1.0.0.0", "172.32.0.1", "fec0::2233", + // 4-in-6 + "::ffff:88.99.100.2", }, ) } func TestIsSpecialNetwork(t *testing.T) { - checkContains(t, IsSpecialNetwork, + checkContains(t, IsSpecialNetwork, AddrIsSpecialNetwork, []string{ // included + "0.0.0.0", + "0.2.0.8", "192.0.2.1", "192.0.2.44", "2001:db8:85a3:8d3:1319:8a2e:370:7348", "255.255.255.255", "224.0.0.22", // IPv4 multicast "ff05::1:3", // IPv6 multicast + // 4-in-6 + "::ffff:255.255.255.255", + "::ffff:192.0.2.1", }, []string{ // excluded "192.0.3.1", @@ -115,15 +127,21 @@ func TestIsSpecialNetwork(t *testing.T) { ) } -func checkContains(t *testing.T, fn func(net.IP) bool, inc, exc []string) { +func checkContains(t *testing.T, fn func(net.IP) bool, fn2 func(netip.Addr) bool, inc, exc []string) { for _, s := range inc { if !fn(parseIP(s)) { - t.Error("returned false for included address", s) + t.Error("returned false for included net.IP", s) + } + if !fn2(netip.MustParseAddr(s)) { + t.Error("returned false for included netip.Addr", s) } } for _, s := range exc { if fn(parseIP(s)) { - t.Error("returned true for excluded address", s) + t.Error("returned true for excluded net.IP", s) + } + if fn2(netip.MustParseAddr(s)) { + t.Error("returned true for excluded netip.Addr", s) } } } @@ -244,14 +262,22 @@ func TestDistinctNetSet(t *testing.T) { } func TestDistinctNetSetAddRemove(t *testing.T) { - cfg := &quick.Config{} - fn := func(ips []net.IP) bool { + cfg := &quick.Config{ + Values: func(s []reflect.Value, rng *rand.Rand) { + slice := make([]netip.Addr, rng.Intn(20)+1) + for i := range slice { + slice[i] = RandomAddr(rng, false) + } + s[0] = reflect.ValueOf(slice) + }, + } + fn := func(ips []netip.Addr) bool { s := DistinctNetSet{Limit: 3, Subnet: 2} for _, ip := range ips { - s.Add(ip) + s.AddAddr(ip) } for _, ip := range ips { - s.Remove(ip) + s.RemoveAddr(ip) } return s.Len() == 0 } diff --git a/p2p/server.go b/p2p/server.go index 13eebed3f4..22e5f6cb94 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -905,14 +905,14 @@ func (srv *Server) listenLoop() { break } - remoteIP := netutil.AddrIP(fd.RemoteAddr()) + remoteIP := netutil.AddrAddr(fd.RemoteAddr()) if err := srv.checkInboundConn(remoteIP); err != nil { srv.log.Debug("Rejected inbound connection", "addr", fd.RemoteAddr(), "err", err) fd.Close() slots <- struct{}{} continue } - if remoteIP != nil { + if remoteIP.IsValid() { fd = newMeteredConn(fd) serveMeter.Mark(1) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) @@ -924,18 +924,19 @@ func (srv *Server) listenLoop() { } } -func (srv *Server) checkInboundConn(remoteIP net.IP) error { - if remoteIP == nil { +func (srv *Server) checkInboundConn(remoteIP netip.Addr) error { + if !remoteIP.IsValid() { + // This case happens for internal test connections without remote address. return nil } // Reject connections that do not match NetRestrict. - if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { + if srv.NetRestrict != nil && !srv.NetRestrict.ContainsAddr(remoteIP) { return errors.New("not in netrestrict list") } // Reject Internet peers that try too often. now := srv.clock.Now() srv.inboundHistory.expire(now, nil) - if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { + if !netutil.AddrIsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { return errors.New("too many attempts") } srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) @@ -1108,7 +1109,7 @@ func (srv *Server) NodeInfo() *NodeInfo { Name: srv.Name, Enode: node.URLv4(), ID: node.ID().String(), - IP: node.IP().String(), + IP: node.IPAddr().String(), ListenAddr: srv.ListenAddr, Protocols: make(map[string]interface{}), } diff --git a/p2p/server_nat_test.go b/p2p/server_nat_test.go index de935fcfc5..cbb1f37e0a 100644 --- a/p2p/server_nat_test.go +++ b/p2p/server_nat_test.go @@ -18,6 +18,7 @@ package p2p import ( "net" + "net/netip" "sync/atomic" "testing" "time" @@ -64,8 +65,8 @@ func TestServerPortMapping(t *testing.T) { t.Error("wrong request count:", reqCount) } enr := srv.LocalNode().Node() - if enr.IP().String() != "192.0.2.0" { - t.Error("wrong IP in ENR:", enr.IP()) + if enr.IPAddr() != netip.MustParseAddr("192.0.2.0") { + t.Error("wrong IP in ENR:", enr.IPAddr()) } if enr.TCP() != 30000 { t.Error("wrong TCP port in ENR:", enr.TCP()) From 6c518fe6068726679016123d574a0523b2855168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Thu, 6 Jun 2024 09:52:57 +0200 Subject: [PATCH 16/19] build: upgrade -dlgo version to Go 1.22.4 (#29938) --- build/checksums.txt | 89 ++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 41 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 94cac9738c..d099e53156 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,48 +5,55 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v2.1.0/ ca89c76851b0900bfcc3cbb9a26cbece1f3d7c64a3bed38723e914713290df6c fixtures_develop.tar.gz -# version:golang 1.22.3 +# version:golang 1.22.4 # https://go.dev/dl/ -80648ef34f903193d72a59c0dff019f5f98ae0c9aa13ade0b0ecbff991a76f68 go1.22.3.src.tar.gz -adc9f5fee89cd53d907eb542d3b269d9d8a08a66bf1ab42175450ffbb58733fb go1.22.3.aix-ppc64.tar.gz -610e48c1df4d2f852de8bc2e7fd2dc1521aac216f0c0026625db12f67f192024 go1.22.3.darwin-amd64.tar.gz -02abeab3f4b8981232237ebd88f0a9bad933bc9621791cd7720a9ca29eacbe9d go1.22.3.darwin-arm64.tar.gz -a5b3d54905f17af2ceaf7fcfe92edee67a5bd4eccd962dd89df719ace3e0894d go1.22.3.dragonfly-amd64.tar.gz -b9989ca87695ae93bacde6f3aa7b13cde5f3825515eb9ed9bbef014273739889 go1.22.3.freebsd-386.tar.gz -7483961fae29d7d768afd5c9c0f229354ca3263ab7119c20bc182761f87cbc74 go1.22.3.freebsd-amd64.tar.gz -edf1f0b8ecf68b14faeedb4f5d868a58c4777a0282bd85e5115c39c010cd0130 go1.22.3.freebsd-arm.tar.gz -572eb70e5e835fbff7d53ebf473f611d7eb458c428f8dbd98a49196883c3309e go1.22.3.freebsd-arm64.tar.gz -ef94eb2b74402e436dce970584222c4e454eb3093908591149bd2ded6862b8af go1.22.3.freebsd-riscv64.tar.gz -3c3f498c68334cbd11f72aadfb6bcb507eb8436cebc50f437a0523cd4c5e03d1 go1.22.3.illumos-amd64.tar.gz -fefba30bb0d3dd1909823ee38c9f1930c3dc5337a2ac4701c2277a329a386b57 go1.22.3.linux-386.tar.gz -8920ea521bad8f6b7bc377b4824982e011c19af27df88a815e3586ea895f1b36 go1.22.3.linux-amd64.tar.gz -6c33e52a5b26e7aa021b94475587fce80043a727a54ceb0eee2f9fc160646434 go1.22.3.linux-arm64.tar.gz -f2bacad20cd2b96f23a86d4826525d42b229fd431cc6d0dec61ff3bc448ef46e go1.22.3.linux-armv6l.tar.gz -41e9328340544893482b2928ae18a9a88ba18b2fdd29ac77f4d33cf1815bbdc2 go1.22.3.linux-loong64.tar.gz -cf4d5faff52e642492729eaf396968f43af179518be769075b90bc1bf650abf6 go1.22.3.linux-mips.tar.gz -3bd009fe2e3d2bfd52433a11cb210d1dfa50b11b4c347a293951efd9e36de945 go1.22.3.linux-mips64.tar.gz -5913b82a042188ef698f7f2dfd0cd0c71f0508a4739de9e41fceff3f4dc769b4 go1.22.3.linux-mips64le.tar.gz -441afebca555be5313867b4577f237c7b5c0fff4386e22e47875b9f805abbec5 go1.22.3.linux-mipsle.tar.gz -f3b53190a76f4a35283501ba6d94cbb72093be0c62ff735c6f9e586a1c983381 go1.22.3.linux-ppc64.tar.gz -04b7b05283de30dd2da20bf3114b2e22cc727938aed3148babaf35cc951051ac go1.22.3.linux-ppc64le.tar.gz -d4992d4a85696e3f1de06cefbfc2fd840c9c6695d77a0f35cfdc4e28b2121c20 go1.22.3.linux-riscv64.tar.gz -2aba796417a69be5f3ed489076bac79c1c02b36e29422712f9f3bf51da9cf2d4 go1.22.3.linux-s390x.tar.gz -d6e6113542dd9f23db899e177fe23772bac114a5ea5e8ee436b9da68628335a8 go1.22.3.netbsd-386.tar.gz -c33cee3075bd18ceefddd75bafa8efb51fbdc17b5ee74275122e7a927a237a4c go1.22.3.netbsd-amd64.tar.gz -1ab251df3c85f3b391a09565ca52fb6e1306527d72852d553e9ab74eabb4ecf8 go1.22.3.netbsd-arm.tar.gz -1d194fe53f5d82f9a612f848950d8af8cab7cb40ccc03f10c4eb1c9808ff1a0c go1.22.3.netbsd-arm64.tar.gz -91d6601727f08506e938640885d3ded784925045e3a4444fd9b4b936efe1b1e0 go1.22.3.openbsd-386.tar.gz -09d0c91ae35a4eea92615426992062ca236cc2f66444fb0b0a24cd3b13bd5297 go1.22.3.openbsd-amd64.tar.gz -338da30cc2c97b9458e0b4caa2509f67bba55d3de16fb7d31775baca82d2e3dc go1.22.3.openbsd-arm.tar.gz -53eadfabd2b7dd09a64941421afee2a2888e2a4f94f353b27919b1dad1171a21 go1.22.3.openbsd-arm64.tar.gz -8a1a2842ae8dcf2374bb05dff58074b368bb698dc9c211c794c1ff119cd9fdc7 go1.22.3.plan9-386.tar.gz -f9816d3dd9e730cad55085ea08c1f0c925720728f9c945fff59cd24d2ac2db7b go1.22.3.plan9-amd64.tar.gz -f4d3d7b17c9e1b1635fcb287b5b5ab5b60acc9db3ba6a27f2b2f5d6537a2ef95 go1.22.3.plan9-arm.tar.gz -46b7999ee94d91b21ad6940b5a3131ff6fe53ef97be9a34e582e2a3ad7263e95 go1.22.3.solaris-amd64.tar.gz -f60f63b8a0885e0d924f39fd284aee5438fe87d8c3d8545a312adf43e0d9edac go1.22.3.windows-386.zip -cab2af6951a6e2115824263f6df13ff069c47270f5788714fa1d776f7f60cb39 go1.22.3.windows-amd64.zip -40b37f4b068fc759f3a0dd61176a0f7570a4ba48bed8561c31d3967a3583981a go1.22.3.windows-arm.zip -59b76ee22b9b1c3afbf7f50e3cb4edb954d6c0d25e5e029ab5483a6804d61e71 go1.22.3.windows-arm64.zip +fed720678e728a7ca30ba8d1ded1caafe27d16028fab0232b8ba8e22008fb784 go1.22.4.src.tar.gz +b9647fa9fc83a0cc5d4f092a19eaeaecf45f063a5aa7d4962fde65aeb7ae6ce1 go1.22.4.aix-ppc64.tar.gz +7788f40f3a46f201df1dc46ca640403eb535d5513fc33449164a90dbd229b761 go1.22.4.darwin-amd64.pkg +c95967f50aa4ace34af0c236cbdb49a9a3e80ee2ad09d85775cb4462a5c19ed3 go1.22.4.darwin-amd64.tar.gz +4036c88faf57a6b096916f1827edcdbf5290a47cc5f59956e88cdd9b1b71088c go1.22.4.darwin-arm64.pkg +242b78dc4c8f3d5435d28a0d2cec9b4c1aa999b601fb8aa59fb4e5a1364bf827 go1.22.4.darwin-arm64.tar.gz +f2fbb51af4719d3616efb482d6ed2b96579b474156f85a7ddc6f126764feec4b go1.22.4.dragonfly-amd64.tar.gz +7c54884bb9f274884651d41e61d1bc12738863ad1497e97ea19ad0e9aa6bf7b5 go1.22.4.freebsd-386.tar.gz +88d44500e1701dd35797619774d6dd51bf60f45a8338b0a82ddc018e4e63fb78 go1.22.4.freebsd-amd64.tar.gz +3d9efe47db142a22679aba46b1772e3900b0d87ae13bd2b3bc80dbf2ac0b2cd6 go1.22.4.freebsd-arm.tar.gz +726dc093cf020277be45debf03c3b02b43c2efb3e2a5d4fba8f52579d65327dc go1.22.4.freebsd-arm64.tar.gz +5f6b67e5e32f1d6ccb2d4dcb44934a5e2e870a877ba7443d86ec43cfc28afa71 go1.22.4.freebsd-riscv64.tar.gz +d56ecc2f85b6418a21ef83879594d0c42ab4f65391a676bb12254870e6690d63 go1.22.4.illumos-amd64.tar.gz +47a2a8d249a91eb8605c33bceec63aedda0441a43eac47b4721e3975ff916cec go1.22.4.linux-386.tar.gz +ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d go1.22.4.linux-amd64.tar.gz +a8e177c354d2e4a1b61020aca3562e27ea3e8f8247eca3170e3fa1e0c2f9e771 go1.22.4.linux-arm64.tar.gz +e2b143fbacbc9cbd448e9ef41ac3981f0488ce849af1cf37e2341d09670661de go1.22.4.linux-armv6l.tar.gz +e2ff9436e4b34bf6926b06d97916e26d67a909a2effec17967245900f0816f1d go1.22.4.linux-loong64.tar.gz +73f0dcc60458c4770593b05a7bc01cc0d31fc98f948c0c2334812c7a1f2fc3f1 go1.22.4.linux-mips.tar.gz +417af97fc2630a647052375768be4c38adcc5af946352ea5b28613ea81ca5d45 go1.22.4.linux-mips64.tar.gz +7486e2d7dd8c98eb44df815ace35a7fe7f30b7c02326e3741bd934077508139b go1.22.4.linux-mips64le.tar.gz +69479c8aad301e459a8365b40cad1074a0dbba5defb9291669f94809c4c4be6e go1.22.4.linux-mipsle.tar.gz +dd238847e65bc3e2745caca475a5db6522a2fcf85cf6c38fc36a06642b19efd7 go1.22.4.linux-ppc64.tar.gz +a3e5834657ef92523f570f798fed42f1f87bc18222a16815ec76b84169649ec4 go1.22.4.linux-ppc64le.tar.gz +56a827ff7dc6245bcd7a1e9288dffaa1d8b0fd7468562264c1523daf3b4f1b4a go1.22.4.linux-riscv64.tar.gz +7590c3e278e2dc6040aae0a39da3ca1eb2e3921673a7304cc34d588c45889eec go1.22.4.linux-s390x.tar.gz +ddd2eebe34471a2502de6c5dad04ab27c9fc80cbde7a9ad5b3c66ecec4504e1d go1.22.4.netbsd-386.tar.gz +33af79f6f935f6fbacc5d23876450b3567b79348fc065beef8e64081127dd234 go1.22.4.netbsd-amd64.tar.gz +fa3550ebd5375a70b3bcd342b5a71f4bd271dcbbfaf4eabefa2144ab5d8924b6 go1.22.4.netbsd-arm.tar.gz +c9a2971dec9f6d320c6f2b049b2353c6d0a2d35e87b8a4b2d78a2f0d62545f8e go1.22.4.netbsd-arm64.tar.gz +d21af022331bfdc2b5b161d616c3a1a4573d33cf7a30416ee509a8f3641deb47 go1.22.4.openbsd-386.tar.gz +72c0094c43f7e5722ec49c2a3e9dfa7a1123ac43a5f3a63eecf3e3795d3ff0ae go1.22.4.openbsd-amd64.tar.gz +1096831ea3c5ea3ca57d14251d9eda3786889531eb40d7d6775dcaa324d4b065 go1.22.4.openbsd-arm.tar.gz +a7ab8d4e0b02bf06ed144ba42c61c0e93ee00f2b433415dfd4ad4b6e79f31650 go1.22.4.openbsd-arm64.tar.gz +9716327c8a628358798898dc5148c49dbbeb5196bf2cbf088e550721a6e4f60b go1.22.4.openbsd-ppc64.tar.gz +a8dd4503c95c32a502a616ab78870a19889c9325fe9bd31eb16dd69346e4bfa8 go1.22.4.plan9-386.tar.gz +5423a25808d76fe5aca8607a2e5ac5673abf45446b168cb5e9d8519ee9fe39a1 go1.22.4.plan9-amd64.tar.gz +6af939ad583f5c85c09c53728ab7d38c3cc2b39167562d6c18a07c5c6608b370 go1.22.4.plan9-arm.tar.gz +e8cabe69c03085725afdb32a6f9998191a3e55a747b270d835fd05000d56abba go1.22.4.solaris-amd64.tar.gz +5c6446e2ea80bc6a971d2b34446f16e6517e638b0ff8d3ea229228d1931790b0 go1.22.4.windows-386.msi +aca4e2c37278a10f1c70dd0df142f7d66b50334fcee48978d409202d308d6d25 go1.22.4.windows-386.zip +3c21105d7b584759b6e266383b777caf6e87142d304a10b539dbc66ab482bb5f go1.22.4.windows-amd64.msi +26321c4d945a0035d8a5bc4a1965b0df401ff8ceac66ce2daadabf9030419a98 go1.22.4.windows-amd64.zip +c4303f02b864304eb83dd1db0b4ebf9d2ec9d216e7ef44a7657b166a52889c7f go1.22.4.windows-arm.msi +5fcd0671a49cecf39b41021621ee1b6e7aa1370f37122b72e80d4fd4185833b6 go1.22.4.windows-arm.zip +553cc6c460f4e3eb4fad5b897c0bb22cd8bbeb20929f0e3eeb939420320292ce go1.22.4.windows-arm64.msi +8a2daa9ea28cbdafddc6171aefed384f4e5b6e714fb52116fe9ed25a132f37ed go1.22.4.windows-arm64.zip # version:golangci 1.59.0 # https://github.com/golangci/golangci-lint/releases/ From cbbfa3eac05b816a704ef2e50ab0a7ac8a9c4dcc Mon Sep 17 00:00:00 2001 From: Marquis Shanahan <29431502+9547@users.noreply.github.com> Date: Thu, 6 Jun 2024 15:55:38 +0800 Subject: [PATCH 17/19] rlp: no need to repeat called len method (#29936) rlp: no need to repeat calling len --- rlp/raw.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/rlp/raw.go b/rlp/raw.go index 773aa7e614..879e3bfe5d 100644 --- a/rlp/raw.go +++ b/rlp/raw.go @@ -30,33 +30,33 @@ var rawValueType = reflect.TypeOf(RawValue{}) // StringSize returns the encoded size of a string. func StringSize(s string) uint64 { - switch { - case len(s) == 0: + switch n := len(s); n { + case 0: return 1 - case len(s) == 1: + case 1: if s[0] <= 0x7f { return 1 } else { return 2 } default: - return uint64(headsize(uint64(len(s))) + len(s)) + return uint64(headsize(uint64(n)) + n) } } // BytesSize returns the encoded size of a byte slice. func BytesSize(b []byte) uint64 { - switch { - case len(b) == 0: + switch n := len(b); n { + case 0: return 1 - case len(b) == 1: + case 1: if b[0] <= 0x7f { return 1 } else { return 2 } default: - return uint64(headsize(uint64(len(b))) + len(b)) + return uint64(headsize(uint64(n)) + n) } } @@ -105,18 +105,20 @@ func SplitUint64(b []byte) (x uint64, rest []byte, err error) { if err != nil { return 0, b, err } - switch { - case len(content) == 0: + switch n := len(content); n { + case 0: return 0, rest, nil - case len(content) == 1: + case 1: if content[0] == 0 { return 0, b, ErrCanonInt } return uint64(content[0]), rest, nil - case len(content) > 8: - return 0, b, errUintOverflow default: - x, err = readSize(content, byte(len(content))) + if n > 8 { + return 0, b, errUintOverflow + } + + x, err = readSize(content, byte(n)) if err != nil { return 0, b, ErrCanonInt } From 0750cb0c8f384ade3213ac3eb18dde214fcc1864 Mon Sep 17 00:00:00 2001 From: Hteev Oli Date: Thu, 6 Jun 2024 15:56:41 +0800 Subject: [PATCH 18/19] p2p/netutil: fix comments (#29942) --- p2p/netutil/net.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/netutil/net.go b/p2p/netutil/net.go index 474912978e..7d8da88670 100644 --- a/p2p/netutil/net.go +++ b/p2p/netutil/net.go @@ -288,12 +288,12 @@ func (s *DistinctNetSet) RemoveAddr(ip netip.Addr) { } } -// Contains whether the given IP is contained in the set. +// Contains reports whether the given IP is contained in the set. func (s DistinctNetSet) Contains(ip net.IP) bool { return s.ContainsAddr(IPToAddr(ip)) } -// ContainsAddr whether the given IP is contained in the set. +// ContainsAddr reports whether the given IP is contained in the set. func (s DistinctNetSet) ContainsAddr(ip netip.Addr) bool { key := s.key(ip) _, ok := s.members[key] From 85459e14395dcd083076ee356358aec961aa35cc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 6 Jun 2024 15:15:22 +0200 Subject: [PATCH 19/19] p2p/discover: unwrap 4-in-6 UDP source addresses (#29944) Fixes an issue where discovery responses were not recognized. --- p2p/discover/v4_udp.go | 5 +++++ p2p/discover/v5_udp.go | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 6234fd93ca..cca01bd3ce 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -557,6 +557,11 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { } func (t *UDPv4) handlePacket(from netip.AddrPort, buf []byte) error { + // Unwrap IPv4-in-6 source address. + if from.Addr().Is4In6() { + from = netip.AddrPortFrom(netip.AddrFrom4(from.Addr().As4()), from.Port()) + } + rawpacket, fromKey, hash, err := v4wire.Decode(buf) if err != nil { t.log.Debug("Bad discv4 packet", "addr", from, "err", err) diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 4cc21cd9c5..81d94812aa 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -674,6 +674,10 @@ func (t *UDPv5) readLoop() { // dispatchReadPacket sends a packet into the dispatch loop. func (t *UDPv5) dispatchReadPacket(from netip.AddrPort, content []byte) bool { + // Unwrap IPv4-in-6 source address. + if from.Addr().Is4In6() { + from = netip.AddrPortFrom(netip.AddrFrom4(from.Addr().As4()), from.Port()) + } select { case t.packetInCh <- ReadPacket{content, from}: return true