From 6e488c244934ea84062cb35d3af26670e1d20e25 Mon Sep 17 00:00:00 2001 From: ddl Date: Mon, 4 Dec 2023 17:52:55 +0800 Subject: [PATCH 01/15] cmd/evm: fix Env struct json tag (#28635) --- cmd/evm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/evm/README.md b/cmd/evm/README.md index e6c6fe06ad..41d8ced278 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -88,7 +88,7 @@ type Env struct { CurrentTimestamp uint64 `json:"currentTimestamp"` Withdrawals []*Withdrawal `json:"withdrawals"` // optional - CurrentDifficulty *big.Int `json:"currentDifficuly"` + CurrentDifficulty *big.Int `json:"currentDifficulty"` CurrentRandom *big.Int `json:"currentRandom"` CurrentBaseFee *big.Int `json:"currentBaseFee"` ParentDifficulty *big.Int `json:"parentDifficulty"` From f04e5bde7487ce554930187e766164b18c37d867 Mon Sep 17 00:00:00 2001 From: BorkBorked <107079055+BorkBorked@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:53:42 +0100 Subject: [PATCH 02/15] accounts/abi/bind: fixed typos (#28634) * Update auth.go * Update backend.go * Update bind.go * Update bind_test.go --- accounts/abi/bind/auth.go | 2 +- accounts/abi/bind/backend.go | 2 +- accounts/abi/bind/bind.go | 2 +- accounts/abi/bind/bind_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 494dc88a57..91913ec3b2 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -56,7 +56,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { } // NewKeyStoreTransactor is a utility method to easily create a transaction signer from -// an decrypted key from a keystore. +// a decrypted key from a keystore. // // Deprecated: Use NewKeyStoreTransactorWithChainID instead. func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) { diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index d13b919641..2e45e86ae2 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -75,7 +75,7 @@ type BlockHashContractCaller interface { // CodeAtHash returns the code of the given account in the state at the specified block hash. CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error) - // CallContractAtHash executes an Ethereum contract all against the state at the specified block hash. + // CallContractAtHash executes an Ethereum contract call against the state at the specified block hash. CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error) } diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 8a54a0e6ef..ec28013463 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -363,7 +363,7 @@ func bindTopicTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { // parameters that are not value types i.e. arrays and structs are not // stored directly but instead a keccak256-hash of an encoding is stored. // - // We only convert stringS and bytes to hash, still need to deal with + // We only convert strings and bytes to hash, still need to deal with // array(both fixed-size and dynamic-size) and struct. if bound == "string" || bound == "[]byte" { bound = "common.Hash" diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 1069f3d396..3191167a00 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1677,7 +1677,7 @@ var bindTests = []struct { } sim.Commit() - // This test the existence of the free retreiver call for view and pure functions + // This test the existence of the free retriever call for view and pure functions if num, err := pav.PureFunc(nil); err != nil { t.Fatalf("Failed to call anonymous field retriever: %v", err) } else if num.Cmp(big.NewInt(42)) != 0 { From 70fd0b635e4198fe8695ff14f62c6736e71b5f27 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 4 Dec 2023 17:55:17 +0530 Subject: [PATCH 03/15] eth/fetcher: fix invalid tracking of received at time for block (#28637) eth/fetcher: fix invalid tracking of received at time --- eth/fetcher/block_fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 8751c4e3ea..126eaaea7f 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -483,7 +483,7 @@ func (f *BlockFetcher) loop() { select { case res := <-resCh: res.Done <- nil - f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now().Add(res.Time)) + f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now()) case <-timeout.C: // The peer didn't respond in time. The request From 2e13b01046aade91e0fe2097a08b761acd413364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Anda=20Estensen?= Date: Mon, 4 Dec 2023 14:55:06 +0100 Subject: [PATCH 04/15] accounts: run tests in parallel (#28544) --- accounts/abi/abi_test.go | 24 ++++++++++++++++ accounts/abi/abifuzzer_test.go | 1 + accounts/abi/bind/backends/simulated_test.go | 29 ++++++++++++++++++++ accounts/abi/bind/base_test.go | 10 +++++++ accounts/abi/bind/bind_test.go | 1 + accounts/abi/bind/util_test.go | 2 ++ accounts/abi/event_test.go | 6 ++++ accounts/abi/method_test.go | 2 ++ accounts/abi/pack_test.go | 5 ++++ accounts/abi/reflect_test.go | 4 +++ accounts/abi/selector_parser_test.go | 1 + accounts/abi/topics_test.go | 9 ++++++ accounts/abi/type_test.go | 5 ++++ accounts/abi/unpack_test.go | 12 ++++++++ accounts/accounts_test.go | 1 + accounts/hd_test.go | 2 ++ accounts/keystore/account_cache_test.go | 3 ++ accounts/keystore/keystore_test.go | 7 +++++ accounts/keystore/passphrase_test.go | 1 + accounts/keystore/plain_test.go | 4 +++ accounts/url_test.go | 5 ++++ 21 files changed, 134 insertions(+) diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 84175df4bb..bc76df0dc2 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -120,6 +120,7 @@ var methods = map[string]Method{ } func TestReader(t *testing.T) { + t.Parallel() abi := ABI{ Methods: methods, } @@ -151,6 +152,7 @@ func TestReader(t *testing.T) { } func TestInvalidABI(t *testing.T) { + t.Parallel() json := `[{ "type" : "function", "name" : "", "constant" : fals }]` _, err := JSON(strings.NewReader(json)) if err == nil { @@ -170,6 +172,7 @@ func TestInvalidABI(t *testing.T) { // constructor(uint256 a, uint256 b) public{} // } func TestConstructor(t *testing.T) { + t.Parallel() json := `[{ "inputs": [{"internalType": "uint256","name": "a","type": "uint256" },{ "internalType": "uint256","name": "b","type": "uint256"}],"stateMutability": "nonpayable","type": "constructor"}]` method := NewMethod("", "", Constructor, "nonpayable", false, false, []Argument{{"a", Uint256, false}, {"b", Uint256, false}}, nil) // Test from JSON @@ -199,6 +202,7 @@ func TestConstructor(t *testing.T) { } func TestTestNumbers(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -236,6 +240,7 @@ func TestTestNumbers(t *testing.T) { } func TestMethodSignature(t *testing.T) { + t.Parallel() m := NewMethod("foo", "foo", Function, "", false, false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil) exp := "foo(string,string)" if m.Sig != exp { @@ -274,6 +279,7 @@ func TestMethodSignature(t *testing.T) { } func TestOverloadedMethodSignature(t *testing.T) { + t.Parallel() json := `[{"constant":true,"inputs":[{"name":"i","type":"uint256"},{"name":"j","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"constant":true,"inputs":[{"name":"i","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"}],"name":"bar","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"},{"indexed":false,"name":"j","type":"uint256"}],"name":"bar","type":"event"}]` abi, err := JSON(strings.NewReader(json)) if err != nil { @@ -297,6 +303,7 @@ func TestOverloadedMethodSignature(t *testing.T) { } func TestCustomErrors(t *testing.T) { + t.Parallel() json := `[{ "inputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ],"name": "MyError", "type": "error"} ]` abi, err := JSON(strings.NewReader(json)) if err != nil { @@ -311,6 +318,7 @@ func TestCustomErrors(t *testing.T) { } func TestMultiPack(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -348,6 +356,7 @@ func ExampleJSON() { } func TestInputVariableInputLength(t *testing.T) { + t.Parallel() const definition = `[ { "type" : "function", "name" : "strOne", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" } ] }, { "type" : "function", "name" : "bytesOne", "constant" : true, "inputs" : [ { "name" : "str", "type" : "bytes" } ] }, @@ -476,6 +485,7 @@ func TestInputVariableInputLength(t *testing.T) { } func TestInputFixedArrayAndVariableInputLength(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Error(err) @@ -650,6 +660,7 @@ func TestInputFixedArrayAndVariableInputLength(t *testing.T) { } func TestDefaultFunctionParsing(t *testing.T) { + t.Parallel() const definition = `[{ "name" : "balance", "type" : "function" }]` abi, err := JSON(strings.NewReader(definition)) @@ -663,6 +674,7 @@ func TestDefaultFunctionParsing(t *testing.T) { } func TestBareEvents(t *testing.T) { + t.Parallel() const definition = `[ { "type" : "event", "name" : "balance" }, { "type" : "event", "name" : "anon", "anonymous" : true}, @@ -739,6 +751,7 @@ func TestBareEvents(t *testing.T) { // // receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestUnpackEvent(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -777,6 +790,7 @@ func TestUnpackEvent(t *testing.T) { } func TestUnpackEventIntoMap(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -827,6 +841,7 @@ func TestUnpackEventIntoMap(t *testing.T) { } func TestUnpackMethodIntoMap(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[],"name":"send","outputs":[{"name":"amount","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"}],"name":"get","outputs":[{"name":"hash","type":"bytes"}],"payable":true,"stateMutability":"payable","type":"function"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -877,6 +892,7 @@ func TestUnpackMethodIntoMap(t *testing.T) { } func TestUnpackIntoMapNamingConflict(t *testing.T) { + t.Parallel() // Two methods have the same name var abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"get","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[],"name":"send","outputs":[{"name":"amount","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"}],"name":"get","outputs":[{"name":"hash","type":"bytes"}],"payable":true,"stateMutability":"payable","type":"function"}]` abi, err := JSON(strings.NewReader(abiJSON)) @@ -960,6 +976,7 @@ func TestUnpackIntoMapNamingConflict(t *testing.T) { } func TestABI_MethodById(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -992,6 +1009,7 @@ func TestABI_MethodById(t *testing.T) { } func TestABI_EventById(t *testing.T) { + t.Parallel() tests := []struct { name string json string @@ -1058,6 +1076,7 @@ func TestABI_EventById(t *testing.T) { } func TestABI_ErrorByID(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(`[ {"inputs":[{"internalType":"uint256","name":"x","type":"uint256"}],"name":"MyError1","type":"error"}, {"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"string","name":"b","type":"string"},{"internalType":"address","name":"c","type":"address"}],"internalType":"struct MyError.MyStruct","name":"x","type":"tuple"},{"internalType":"address","name":"y","type":"address"},{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"string","name":"b","type":"string"},{"internalType":"address","name":"c","type":"address"}],"internalType":"struct MyError.MyStruct","name":"z","type":"tuple"}],"name":"MyError2","type":"error"}, @@ -1088,6 +1107,7 @@ func TestABI_ErrorByID(t *testing.T) { // TestDoubleDuplicateMethodNames checks that if transfer0 already exists, there won't be a name // conflict and that the second transfer method will be renamed transfer1. func TestDoubleDuplicateMethodNames(t *testing.T) { + t.Parallel() abiJSON := `[{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"}],"name":"transfer0","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"},{"name":"customFallback","type":"string"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1117,6 +1137,7 @@ func TestDoubleDuplicateMethodNames(t *testing.T) { // event send(); // } func TestDoubleDuplicateEventNames(t *testing.T) { + t.Parallel() abiJSON := `[{"anonymous": false,"inputs": [{"indexed": false,"internalType": "uint256","name": "a","type": "uint256"}],"name": "send","type": "event"},{"anonymous": false,"inputs": [],"name": "send0","type": "event"},{ "anonymous": false, "inputs": [],"name": "send","type": "event"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1144,6 +1165,7 @@ func TestDoubleDuplicateEventNames(t *testing.T) { // event send(uint256, uint256); // } func TestUnnamedEventParam(t *testing.T) { + t.Parallel() abiJSON := `[{ "anonymous": false, "inputs": [{ "indexed": false,"internalType": "uint256", "name": "","type": "uint256"},{"indexed": false,"internalType": "uint256","name": "","type": "uint256"}],"name": "send","type": "event"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1177,7 +1199,9 @@ func TestUnpackRevert(t *testing.T) { {"4e487b7100000000000000000000000000000000000000000000000000000000000000ff", "unknown panic code: 0xff", nil}, } for index, c := range cases { + index, c := index, c t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) { + t.Parallel() got, err := UnpackRevert(common.Hex2Bytes(c.input)) if c.expectErr != nil { if err == nil { diff --git a/accounts/abi/abifuzzer_test.go b/accounts/abi/abifuzzer_test.go index 4b67947815..dbf6ab6c54 100644 --- a/accounts/abi/abifuzzer_test.go +++ b/accounts/abi/abifuzzer_test.go @@ -28,6 +28,7 @@ import ( // TestReplicate can be used to replicate crashers from the fuzzing tests. // Just replace testString with the data in .quoted func TestReplicate(t *testing.T) { + t.Parallel() //t.Skip("Test only useful for reproducing issues") fuzzAbi([]byte("\x20\x20\x20\x20\x20\x20\x20\x20\x80\x00\x00\x00\x20\x20\x20\x20\x00")) //fuzzAbi([]byte("asdfasdfkadsf;lasdf;lasd;lfk")) diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index a41d168411..a2acf7ead5 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -38,6 +38,7 @@ import ( ) func TestSimulatedBackend(t *testing.T) { + t.Parallel() var gasLimit uint64 = 8000029 key, _ := crypto.GenerateKey() // nolint: gosec auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -121,6 +122,7 @@ func simTestBackend(testAddr common.Address) *SimulatedBackend { } func TestNewSimulatedBackend(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) expectedBal := big.NewInt(10000000000000000) sim := simTestBackend(testAddr) @@ -142,6 +144,7 @@ func TestNewSimulatedBackend(t *testing.T) { } func TestAdjustTime(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -159,6 +162,7 @@ func TestAdjustTime(t *testing.T) { } func TestNewAdjustTimeFail(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.blockchain.Stop() @@ -202,6 +206,7 @@ func TestNewAdjustTimeFail(t *testing.T) { } func TestBalanceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) expectedBal := big.NewInt(10000000000000000) sim := simTestBackend(testAddr) @@ -219,6 +224,7 @@ func TestBalanceAt(t *testing.T) { } func TestBlockByHash(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -240,6 +246,7 @@ func TestBlockByHash(t *testing.T) { } func TestBlockByNumber(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -275,6 +282,7 @@ func TestBlockByNumber(t *testing.T) { } func TestNonceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -328,6 +336,7 @@ func TestNonceAt(t *testing.T) { } func TestSendTransaction(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -362,6 +371,7 @@ func TestSendTransaction(t *testing.T) { } func TestTransactionByHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := NewSimulatedBackend( @@ -416,6 +426,7 @@ func TestTransactionByHash(t *testing.T) { } func TestEstimateGas(t *testing.T) { + t.Parallel() /* pragma solidity ^0.6.4; contract GasEstimation { @@ -535,6 +546,7 @@ func TestEstimateGas(t *testing.T) { } func TestEstimateGasWithPrice(t *testing.T) { + t.Parallel() key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) @@ -625,6 +637,7 @@ func TestEstimateGasWithPrice(t *testing.T) { } func TestHeaderByHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -646,6 +659,7 @@ func TestHeaderByHash(t *testing.T) { } func TestHeaderByNumber(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -692,6 +706,7 @@ func TestHeaderByNumber(t *testing.T) { } func TestTransactionCount(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -744,6 +759,7 @@ func TestTransactionCount(t *testing.T) { } func TestTransactionInBlock(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -809,6 +825,7 @@ func TestTransactionInBlock(t *testing.T) { } func TestPendingNonceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -874,6 +891,7 @@ func TestPendingNonceAt(t *testing.T) { } func TestTransactionReceipt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -908,6 +926,7 @@ func TestTransactionReceipt(t *testing.T) { } func TestSuggestGasPrice(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, @@ -924,6 +943,7 @@ func TestSuggestGasPrice(t *testing.T) { } func TestPendingCodeAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -960,6 +980,7 @@ func TestPendingCodeAt(t *testing.T) { } func TestCodeAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -997,6 +1018,7 @@ func TestCodeAt(t *testing.T) { } func TestCodeAtHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1037,6 +1059,7 @@ func TestCodeAtHash(t *testing.T) { // // receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestPendingAndCallContract(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1138,6 +1161,7 @@ contract Reverter { } }*/ func TestCallContractRevert(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1233,6 +1257,7 @@ func TestCallContractRevert(t *testing.T) { // Since Commit() was called 2n+1 times in total, // having a chain length of just n+1 means that a reorg occurred. func TestFork(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1286,6 +1311,7 @@ const callableBin = "6080604052348015600f57600080fd5b5060998061001e6000396000f3f // 9. Re-send the transaction and mine a block. // 10. Check that the event was reborn. func TestForkLogsReborn(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1359,6 +1385,7 @@ func TestForkLogsReborn(t *testing.T) { // 5. Mine a block, Re-send the transaction and mine another one. // 6. Check that the TX is now included in block 2. func TestForkResendTx(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1395,6 +1422,7 @@ func TestForkResendTx(t *testing.T) { } func TestCommitReturnValue(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1436,6 +1464,7 @@ func TestCommitReturnValue(t *testing.T) { // TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork // block's parent rather than the canonical head's parent. func TestAdjustTimeAfterFork(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index 44552ab121..f7eb7d14d3 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -135,6 +135,7 @@ func (mc *mockBlockHashCaller) CallContractAtHash(ctx context.Context, call ethe } func TestPassingBlockNumber(t *testing.T) { + t.Parallel() mc := &mockPendingCaller{ mockCaller: &mockCaller{ codeAtBytes: []byte{1, 2, 3}, @@ -186,6 +187,7 @@ func TestPassingBlockNumber(t *testing.T) { const hexData = "0x000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158" func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) { + t.Parallel() hash := crypto.Keccak256Hash([]byte("testName")) topics := []common.Hash{ crypto.Keccak256Hash([]byte("received(string,address,uint256,bytes)")), @@ -207,6 +209,7 @@ func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) { } func TestUnpackAnonymousLogIntoMap(t *testing.T) { + t.Parallel() mockLog := newMockLog(nil, common.HexToHash("0x0")) abiString := `[{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"received","type":"event"}]` @@ -224,6 +227,7 @@ func TestUnpackAnonymousLogIntoMap(t *testing.T) { } func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) { + t.Parallel() sliceBytes, err := rlp.EncodeToBytes([]string{"name1", "name2", "name3", "name4"}) if err != nil { t.Fatal(err) @@ -249,6 +253,7 @@ func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedArrayTyLogIntoMap(t *testing.T) { + t.Parallel() arrBytes, err := rlp.EncodeToBytes([2]common.Address{common.HexToAddress("0x0"), common.HexToAddress("0x376c47978271565f56DEB45495afa69E59c16Ab2")}) if err != nil { t.Fatal(err) @@ -274,6 +279,7 @@ func TestUnpackIndexedArrayTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedFuncTyLogIntoMap(t *testing.T) { + t.Parallel() mockAddress := common.HexToAddress("0x376c47978271565f56DEB45495afa69E59c16Ab2") addrBytes := mockAddress.Bytes() hash := crypto.Keccak256Hash([]byte("mockFunction(address,uint)")) @@ -300,6 +306,7 @@ func TestUnpackIndexedFuncTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) { + t.Parallel() bytes := []byte{1, 2, 3, 4, 5} hash := crypto.Keccak256Hash(bytes) topics := []common.Hash{ @@ -322,6 +329,7 @@ func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) { } func TestTransactGasFee(t *testing.T) { + t.Parallel() assert := assert.New(t) // GasTipCap and GasFeeCap @@ -397,6 +405,7 @@ func newMockLog(topics []common.Hash, txHash common.Hash) types.Log { } func TestCall(t *testing.T) { + t.Parallel() var method, methodWithArg = "something", "somethingArrrrg" tests := []struct { name, method string @@ -572,6 +581,7 @@ func TestCall(t *testing.T) { // TestCrashers contains some strings which previously caused the abi codec to crash. func TestCrashers(t *testing.T) { + t.Parallel() abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"_1"}]}]}]`)) abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"&"}]}]}]`)) abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"----"}]}]}]`)) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 3191167a00..a5f7afa73c 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -2067,6 +2067,7 @@ var bindTests = []struct { // Tests that packages generated by the binder can be successfully compiled and // the requested tester run against it. func TestGolangBindings(t *testing.T) { + t.Parallel() // Skip the test if no Go command can be found gocmd := runtime.GOROOT() + "/bin/go" if !common.FileExist(gocmd) { diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 16110b5d27..826426632c 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -53,6 +53,7 @@ var waitDeployedTests = map[string]struct { } func TestWaitDeployed(t *testing.T) { + t.Parallel() for name, test := range waitDeployedTests { backend := backends.NewSimulatedBackend( core.GenesisAlloc{ @@ -100,6 +101,7 @@ func TestWaitDeployed(t *testing.T) { } func TestWaitDeployedCornerCases(t *testing.T) { + t.Parallel() backend := backends.NewSimulatedBackend( core.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 8f73419496..fffe28ea63 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -81,6 +81,7 @@ var pledgeData1 = "00000000000000000000000000ce0d46d924cc8437c806721496599fc3ffa var mixedCaseData1 = "00000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000020489e8000000000000000000000000000000000000000000000000000000000000000f4241" func TestEventId(t *testing.T) { + t.Parallel() var table = []struct { definition string expectations map[string]common.Hash @@ -112,6 +113,7 @@ func TestEventId(t *testing.T) { } func TestEventString(t *testing.T) { + t.Parallel() var table = []struct { definition string expectations map[string]string @@ -146,6 +148,7 @@ func TestEventString(t *testing.T) { // TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array. func TestEventMultiValueWithArrayUnpack(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` abi, err := JSON(strings.NewReader(definition)) require.NoError(t, err) @@ -161,6 +164,7 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) { } func TestEventTupleUnpack(t *testing.T) { + t.Parallel() type EventTransfer struct { Value *big.Int } @@ -351,6 +355,7 @@ func unpackTestEventData(dest interface{}, hexData string, jsonEvent []byte, ass // TestEventUnpackIndexed verifies that indexed field will be skipped by event decoder. func TestEventUnpackIndexed(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` type testStruct struct { Value1 uint8 // indexed @@ -368,6 +373,7 @@ func TestEventUnpackIndexed(t *testing.T) { // TestEventIndexedWithArrayUnpack verifies that decoder will not overflow when static array is indexed input. func TestEventIndexedWithArrayUnpack(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"string"}]}]` type testStruct struct { Value1 [2]uint8 // indexed diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go index 9230e307aa..6322173920 100644 --- a/accounts/abi/method_test.go +++ b/accounts/abi/method_test.go @@ -35,6 +35,7 @@ const methoddata = ` ]` func TestMethodString(t *testing.T) { + t.Parallel() var table = []struct { method string expectation string @@ -99,6 +100,7 @@ func TestMethodString(t *testing.T) { } func TestMethodSig(t *testing.T) { + t.Parallel() var cases = []struct { method string expect string diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 5c7cb1cc1a..00bdae469e 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -32,8 +32,11 @@ import ( // TestPack tests the general pack/unpack tests in packing_test.go func TestPack(t *testing.T) { + t.Parallel() for i, test := range packUnpackTests { + i, test := i, test t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() encb, err := hex.DecodeString(test.packed) if err != nil { t.Fatalf("invalid hex %s: %v", test.packed, err) @@ -57,6 +60,7 @@ func TestPack(t *testing.T) { } func TestMethodPack(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -177,6 +181,7 @@ func TestMethodPack(t *testing.T) { } func TestPackNumber(t *testing.T) { + t.Parallel() tests := []struct { value reflect.Value packed []byte diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index 76ef1ad2aa..6c7ae57087 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -170,8 +170,11 @@ var reflectTests = []reflectTest{ } func TestReflectNameToStruct(t *testing.T) { + t.Parallel() for _, test := range reflectTests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() m, err := mapArgNamesToStructFields(test.args, reflect.ValueOf(test.struc)) if len(test.err) > 0 { if err == nil || err.Error() != test.err { @@ -192,6 +195,7 @@ func TestReflectNameToStruct(t *testing.T) { } func TestConvertType(t *testing.T) { + t.Parallel() // Test Basic Struct type T struct { X *big.Int diff --git a/accounts/abi/selector_parser_test.go b/accounts/abi/selector_parser_test.go index f6f134492b..6cb0ae0e70 100644 --- a/accounts/abi/selector_parser_test.go +++ b/accounts/abi/selector_parser_test.go @@ -24,6 +24,7 @@ import ( ) func TestParseSelector(t *testing.T) { + t.Parallel() mkType := func(types ...interface{}) []ArgumentMarshaling { var result []ArgumentMarshaling for i, typeOrComponents := range types { diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index 30cf21d0b8..b31f58fba3 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -26,6 +26,7 @@ import ( ) func TestMakeTopics(t *testing.T) { + t.Parallel() type args struct { query [][]interface{} } @@ -117,7 +118,9 @@ func TestMakeTopics(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() got, err := MakeTopics(tt.args.query...) if (err != nil) != tt.wantErr { t.Errorf("makeTopics() error = %v, wantErr %v", err, tt.wantErr) @@ -347,10 +350,13 @@ func setupTopicsTests() []topicTest { } func TestParseTopics(t *testing.T) { + t.Parallel() tests := setupTopicsTests() for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() createObj := tt.args.createObj() if err := ParseTopics(createObj, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr { t.Errorf("parseTopics() error = %v, wantErr %v", err, tt.wantErr) @@ -364,10 +370,13 @@ func TestParseTopics(t *testing.T) { } func TestParseTopicsIntoMap(t *testing.T) { + t.Parallel() tests := setupTopicsTests() for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() outMap := make(map[string]interface{}) if err := ParseTopicsIntoMap(outMap, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr { t.Errorf("parseTopicsIntoMap() error = %v, wantErr %v", err, tt.wantErr) diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index a72531ba27..ae69872ad8 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -31,6 +31,7 @@ type typeWithoutStringer Type // Tests that all allowed types get recognized by the type parser. func TestTypeRegexp(t *testing.T) { + t.Parallel() tests := []struct { blob string components []ArgumentMarshaling @@ -117,6 +118,7 @@ func TestTypeRegexp(t *testing.T) { } func TestTypeCheck(t *testing.T) { + t.Parallel() for i, test := range []struct { typ string components []ArgumentMarshaling @@ -308,6 +310,7 @@ func TestTypeCheck(t *testing.T) { } func TestInternalType(t *testing.T) { + t.Parallel() components := []ArgumentMarshaling{{Name: "a", Type: "int64"}} internalType := "struct a.b[]" kind := Type{ @@ -332,6 +335,7 @@ func TestInternalType(t *testing.T) { } func TestGetTypeSize(t *testing.T) { + t.Parallel() var testCases = []struct { typ string components []ArgumentMarshaling @@ -368,6 +372,7 @@ func TestGetTypeSize(t *testing.T) { } func TestNewFixedBytesOver32(t *testing.T) { + t.Parallel() _, err := NewType("bytes4096", "", nil) if err == nil { t.Errorf("fixed bytes with size over 32 is not spec'd") diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index a7ee1d9202..29891ec0a4 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -33,6 +33,7 @@ import ( // TestUnpack tests the general pack/unpack tests in packing_test.go func TestUnpack(t *testing.T) { + t.Parallel() for i, test := range packUnpackTests { t.Run(strconv.Itoa(i)+" "+test.def, func(t *testing.T) { //Unpack @@ -224,6 +225,7 @@ var unpackTests = []unpackTest{ // TestLocalUnpackTests runs test specially designed only for unpacking. // All test cases that can be used to test packing and unpacking should move to packing_test.go func TestLocalUnpackTests(t *testing.T) { + t.Parallel() for i, test := range unpackTests { t.Run(strconv.Itoa(i), func(t *testing.T) { //Unpack @@ -251,6 +253,7 @@ func TestLocalUnpackTests(t *testing.T) { } func TestUnpackIntoInterfaceSetDynamicArrayOutput(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(`[{"constant":true,"inputs":[],"name":"testDynamicFixedBytes15","outputs":[{"name":"","type":"bytes15[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"testDynamicFixedBytes32","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"}]`)) if err != nil { t.Fatal(err) @@ -321,6 +324,7 @@ func methodMultiReturn(require *require.Assertions) (ABI, []byte, methodMultiOut } func TestMethodMultiReturn(t *testing.T) { + t.Parallel() type reversed struct { String string Int *big.Int @@ -400,6 +404,7 @@ func TestMethodMultiReturn(t *testing.T) { } func TestMultiReturnWithArray(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -423,6 +428,7 @@ func TestMultiReturnWithArray(t *testing.T) { } func TestMultiReturnWithStringArray(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -453,6 +459,7 @@ func TestMultiReturnWithStringArray(t *testing.T) { } func TestMultiReturnWithStringSlice(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -485,6 +492,7 @@ func TestMultiReturnWithStringSlice(t *testing.T) { } func TestMultiReturnWithDeeplyNestedArray(t *testing.T) { + t.Parallel() // Similar to TestMultiReturnWithArray, but with a special case in mind: // values of nested static arrays count towards the size as well, and any element following // after such nested array argument should be read with the correct offset, @@ -525,6 +533,7 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) { } func TestUnmarshal(t *testing.T) { + t.Parallel() const definition = `[ { "name" : "int", "type": "function", "outputs": [ { "type": "uint256" } ] }, { "name" : "bool", "type": "function", "outputs": [ { "type": "bool" } ] }, @@ -774,6 +783,7 @@ func TestUnmarshal(t *testing.T) { } func TestUnpackTuple(t *testing.T) { + t.Parallel() const simpleTuple = `[{"name":"tuple","type":"function","outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]` abi, err := JSON(strings.NewReader(simpleTuple)) if err != nil { @@ -876,6 +886,7 @@ func TestUnpackTuple(t *testing.T) { } func TestOOMMaliciousInput(t *testing.T) { + t.Parallel() oomTests := []unpackTest{ { def: `[{"type": "uint8[]"}]`, @@ -946,6 +957,7 @@ func TestOOMMaliciousInput(t *testing.T) { } func TestPackAndUnpackIncompatibleNumber(t *testing.T) { + t.Parallel() var encodeABI Arguments uint256Ty, err := NewType("uint256", "", nil) if err != nil { diff --git a/accounts/accounts_test.go b/accounts/accounts_test.go index e8274f9f04..2c4138aa78 100644 --- a/accounts/accounts_test.go +++ b/accounts/accounts_test.go @@ -24,6 +24,7 @@ import ( ) func TestTextHash(t *testing.T) { + t.Parallel() hash := TextHash([]byte("Hello Joe")) want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b") if !bytes.Equal(hash, want) { diff --git a/accounts/hd_test.go b/accounts/hd_test.go index 0743bbe666..118ec5187b 100644 --- a/accounts/hd_test.go +++ b/accounts/hd_test.go @@ -25,6 +25,7 @@ import ( // Tests that HD derivation paths can be correctly parsed into our internal binary // representation. func TestHDPathParsing(t *testing.T) { + t.Parallel() tests := []struct { input string output DerivationPath @@ -89,6 +90,7 @@ func testDerive(t *testing.T, next func() DerivationPath, expected []string) { } func TestHdPathIteration(t *testing.T) { + t.Parallel() testDerive(t, DefaultIterator(DefaultBaseDerivationPath), []string{ "m/44'/60'/0'/0/0", "m/44'/60'/0'/0/1", diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 371d274441..48a238048f 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -152,6 +152,7 @@ func TestWatchNoDir(t *testing.T) { } func TestCacheInitialReload(t *testing.T) { + t.Parallel() cache, _ := newAccountCache(cachetestDir) accounts := cache.accounts() if !reflect.DeepEqual(accounts, cachetestAccounts) { @@ -160,6 +161,7 @@ func TestCacheInitialReload(t *testing.T) { } func TestCacheAddDeleteOrder(t *testing.T) { + t.Parallel() cache, _ := newAccountCache("testdata/no-such-dir") cache.watcher.running = true // prevent unexpected reloads @@ -244,6 +246,7 @@ func TestCacheAddDeleteOrder(t *testing.T) { } func TestCacheFind(t *testing.T) { + t.Parallel() dir := filepath.Join("testdata", "dir") cache, _ := newAccountCache(dir) cache.watcher.running = true // prevent unexpected reloads diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index deb7cae9f9..c9a23eddd6 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -36,6 +36,7 @@ import ( var testSigData = make([]byte, 32) func TestKeyStore(t *testing.T) { + t.Parallel() dir, ks := tmpKeyStore(t, true) a, err := ks.NewAccount("foo") @@ -70,6 +71,7 @@ func TestKeyStore(t *testing.T) { } func TestSign(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) pass := "" // not used but required by API @@ -86,6 +88,7 @@ func TestSign(t *testing.T) { } func TestSignWithPassphrase(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) pass := "passwd" @@ -280,6 +283,7 @@ type walletEvent struct { // Tests that wallet notifications and correctly fired when accounts are added // or deleted from the keystore. func TestWalletNotifications(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, false) // Subscribe to the wallet feed and collect events. @@ -341,6 +345,7 @@ func TestWalletNotifications(t *testing.T) { // TestImportExport tests the import functionality of a keystore. func TestImportECDSA(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) key, err := crypto.GenerateKey() if err != nil { @@ -359,6 +364,7 @@ func TestImportECDSA(t *testing.T) { // TestImportECDSA tests the import and export functionality of a keystore. func TestImportExport(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) acc, err := ks.NewAccount("old") if err != nil { @@ -387,6 +393,7 @@ func TestImportExport(t *testing.T) { // TestImportRace tests the keystore on races. // This test should fail under -race if importing races. func TestImportRace(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) acc, err := ks.NewAccount("old") if err != nil { diff --git a/accounts/keystore/passphrase_test.go b/accounts/keystore/passphrase_test.go index 1de43a96da..20ec0f5519 100644 --- a/accounts/keystore/passphrase_test.go +++ b/accounts/keystore/passphrase_test.go @@ -30,6 +30,7 @@ const ( // Tests that a json key file can be decrypted and encrypted in multiple rounds. func TestKeyEncryptDecrypt(t *testing.T) { + t.Parallel() keyjson, err := os.ReadFile("testdata/very-light-scrypt.json") if err != nil { t.Fatal(err) diff --git a/accounts/keystore/plain_test.go b/accounts/keystore/plain_test.go index 93165d5cd3..737eb7fd61 100644 --- a/accounts/keystore/plain_test.go +++ b/accounts/keystore/plain_test.go @@ -40,6 +40,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) { } func TestKeyStorePlain(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, false) pass := "" // not used but required by API @@ -60,6 +61,7 @@ func TestKeyStorePlain(t *testing.T) { } func TestKeyStorePassphrase(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, true) pass := "foo" @@ -80,6 +82,7 @@ func TestKeyStorePassphrase(t *testing.T) { } func TestKeyStorePassphraseDecryptionFail(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, true) pass := "foo" @@ -93,6 +96,7 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) { } func TestImportPreSaleKey(t *testing.T) { + t.Parallel() dir, ks := tmpKeyStoreIface(t, true) // file content of a presale key file generated with: diff --git a/accounts/url_test.go b/accounts/url_test.go index 52be4c558d..f481a1016d 100644 --- a/accounts/url_test.go +++ b/accounts/url_test.go @@ -21,6 +21,7 @@ import ( ) func TestURLParsing(t *testing.T) { + t.Parallel() url, err := parseURL("https://ethereum.org") if err != nil { t.Errorf("unexpected error: %v", err) @@ -40,6 +41,7 @@ func TestURLParsing(t *testing.T) { } func TestURLString(t *testing.T) { + t.Parallel() url := URL{Scheme: "https", Path: "ethereum.org"} if url.String() != "https://ethereum.org" { t.Errorf("expected: %v, got: %v", "https://ethereum.org", url.String()) @@ -52,6 +54,7 @@ func TestURLString(t *testing.T) { } func TestURLMarshalJSON(t *testing.T) { + t.Parallel() url := URL{Scheme: "https", Path: "ethereum.org"} json, err := url.MarshalJSON() if err != nil { @@ -63,6 +66,7 @@ func TestURLMarshalJSON(t *testing.T) { } func TestURLUnmarshalJSON(t *testing.T) { + t.Parallel() url := &URL{} err := url.UnmarshalJSON([]byte("\"https://ethereum.org\"")) if err != nil { @@ -77,6 +81,7 @@ func TestURLUnmarshalJSON(t *testing.T) { } func TestURLComparison(t *testing.T) { + t.Parallel() tests := []struct { urlA URL urlB URL From 3dc071e0367505f4234ef880b853115c4040cfad Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Dec 2023 09:37:48 +0100 Subject: [PATCH 05/15] eth/tracers/logger: make structlog/json-log stack hex again (#28628) * common/hexutil: define hex wrappers for uint256.Int * eth/tracers/logger: make structlog/json-log stack hex again * common/hexutil: goimports --- common/hexutil/json.go | 45 ++++++++++++++++++++++ common/hexutil/json_test.go | 60 +++++++++++++++++++++++++++++ eth/tracers/logger/gen_structlog.go | 16 ++++++-- eth/tracers/logger/logger.go | 1 + 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/common/hexutil/json.go b/common/hexutil/json.go index 50db208118..e0ac98f52d 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -23,6 +23,8 @@ import ( "math/big" "reflect" "strconv" + + "github.com/holiman/uint256" ) var ( @@ -30,6 +32,7 @@ var ( bigT = reflect.TypeOf((*Big)(nil)) uintT = reflect.TypeOf(Uint(0)) uint64T = reflect.TypeOf(Uint64(0)) + u256T = reflect.TypeOf((*uint256.Int)(nil)) ) // Bytes marshals/unmarshals as a JSON string with 0x prefix. @@ -225,6 +228,48 @@ func (b *Big) UnmarshalGraphQL(input interface{}) error { return err } +// U256 marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +type U256 uint256.Int + +// MarshalText implements encoding.TextMarshaler +func (b U256) MarshalText() ([]byte, error) { + u256 := (*uint256.Int)(&b) + return []byte(u256.Hex()), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *U256) UnmarshalJSON(input []byte) error { + // The uint256.Int.UnmarshalJSON method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + if !isString(input) { + return errNonString(u256T) + } + // The hex decoder needs to accept empty string ("") as '0', which uint256.Int + // would reject. + if len(input) == 2 { + (*uint256.Int)(b).Clear() + return nil + } + err := (*uint256.Int)(b).SetFromHex(string(input[1 : len(input)-1])) + if err != nil { + return &json.UnmarshalTypeError{Value: err.Error(), Type: u256T} + } + return nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (b *U256) UnmarshalText(input []byte) error { + // The uint256.Int.UnmarshalText method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + return (*uint256.Int)(b).SetFromHex(string(input)) +} + +// String returns the hex encoding of b. +func (b *U256) String() string { + return (*uint256.Int)(b).Hex() +} + // Uint64 marshals/unmarshals as a JSON string with 0x prefix. // The zero value marshals as "0x0". type Uint64 uint64 diff --git a/common/hexutil/json_test.go b/common/hexutil/json_test.go index ed7d6fad1a..7cca300951 100644 --- a/common/hexutil/json_test.go +++ b/common/hexutil/json_test.go @@ -23,6 +23,8 @@ import ( "errors" "math/big" "testing" + + "github.com/holiman/uint256" ) func checkError(t *testing.T, input string, got, want error) bool { @@ -176,6 +178,64 @@ func TestUnmarshalBig(t *testing.T) { } } +var unmarshalU256Tests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(u256T)}, + {input: "10", wantErr: errNonString(u256T)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, u256T)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, u256T)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, u256T)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, u256T)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, u256T)}, + { + input: `"0x10000000000000000000000000000000000000000000000000000000000000000"`, + wantErr: wrapTypeError(ErrBig256Range, u256T), + }, + + // valid encoding + {input: `""`, want: big.NewInt(0)}, + {input: `"0x0"`, want: big.NewInt(0)}, + {input: `"0x2"`, want: big.NewInt(0x2)}, + {input: `"0x2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0X2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0x1122aaff"`, want: big.NewInt(0x1122aaff)}, + {input: `"0xbBb"`, want: big.NewInt(0xbbb)}, + {input: `"0xfffffffff"`, want: big.NewInt(0xfffffffff)}, + { + input: `"0x112233445566778899aabbccddeeff"`, + want: referenceBig("112233445566778899aabbccddeeff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, +} + +func TestUnmarshalU256(t *testing.T) { + for _, test := range unmarshalU256Tests { + var v U256 + err := json.Unmarshal([]byte(test.input), &v) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if test.want == nil { + continue + } + want := new(uint256.Int) + want.SetFromBig(test.want.(*big.Int)) + have := (*uint256.Int)(&v) + if want.Cmp(have) != 0 { + t.Errorf("input %s: value mismatch: have %x, want %x", test.input, have, want) + continue + } + } +} + func BenchmarkUnmarshalBig(b *testing.B) { input := []byte(`"0x123456789abcdef123456789abcdef"`) for i := 0; i < b.N; i++ { diff --git a/eth/tracers/logger/gen_structlog.go b/eth/tracers/logger/gen_structlog.go index df06a9ee6b..b406cb3445 100644 --- a/eth/tracers/logger/gen_structlog.go +++ b/eth/tracers/logger/gen_structlog.go @@ -23,7 +23,7 @@ func (s StructLog) MarshalJSON() ([]byte, error) { GasCost math.HexOrDecimal64 `json:"gasCost"` Memory hexutil.Bytes `json:"memory,omitempty"` MemorySize int `json:"memSize"` - Stack []uint256.Int `json:"stack"` + Stack []hexutil.U256 `json:"stack"` ReturnData hexutil.Bytes `json:"returnData,omitempty"` Storage map[common.Hash]common.Hash `json:"-"` Depth int `json:"depth"` @@ -39,7 +39,12 @@ func (s StructLog) MarshalJSON() ([]byte, error) { enc.GasCost = math.HexOrDecimal64(s.GasCost) enc.Memory = s.Memory enc.MemorySize = s.MemorySize - enc.Stack = s.Stack + if s.Stack != nil { + enc.Stack = make([]hexutil.U256, len(s.Stack)) + for k, v := range s.Stack { + enc.Stack[k] = hexutil.U256(v) + } + } enc.ReturnData = s.ReturnData enc.Storage = s.Storage enc.Depth = s.Depth @@ -59,7 +64,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { GasCost *math.HexOrDecimal64 `json:"gasCost"` Memory *hexutil.Bytes `json:"memory,omitempty"` MemorySize *int `json:"memSize"` - Stack []uint256.Int `json:"stack"` + Stack []hexutil.U256 `json:"stack"` ReturnData *hexutil.Bytes `json:"returnData,omitempty"` Storage map[common.Hash]common.Hash `json:"-"` Depth *int `json:"depth"` @@ -89,7 +94,10 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { s.MemorySize = *dec.MemorySize } if dec.Stack != nil { - s.Stack = dec.Stack + s.Stack = make([]uint256.Int, len(dec.Stack)) + for k, v := range dec.Stack { + s.Stack[k] = uint256.Int(v) + } } if dec.ReturnData != nil { s.ReturnData = *dec.ReturnData diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 4c9b910a27..2b36f9f492 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -83,6 +83,7 @@ type structLogMarshaling struct { GasCost math.HexOrDecimal64 Memory hexutil.Bytes ReturnData hexutil.Bytes + Stack []hexutil.U256 OpName string `json:"opName"` // adds call to OpName() in MarshalJSON ErrorString string `json:"error,omitempty"` // adds call to ErrorString() in MarshalJSON } From b8d44ed98b9bc8320f1f64e82c13cf32918f1f6b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Dec 2023 11:54:44 +0100 Subject: [PATCH 06/15] log: remove lazy, remove unused interfaces, unexport methods (#28622) This change - Removes interface `log.Format`, - Removes method `log.FormatFunc`, - unexports `TerminalHandler.TerminalFormat` formatting methods (renamed to `TerminalHandler.format`) - removes the notion of `log.Lazy` values The lazy handler was useful in the old log package, since it could defer the evaluation of costly attributes until later in the log pipeline: thus, if the logging was done at 'Trace', we could skip evaluation if logging only was set to 'Info'. With the move to slog, this way of deferring evaluation is no longer needed, since slog introduced 'Enabled': the caller can thus do the evaluate-or-not decision at the callsite, which is much more straight-forward than dealing with lazy reflect-based evaluation. Also, lazy evaluation would not work with 'native' slog, as in, these two statements would be evaluated differently: ```golang log.Info("foo", "my lazy", lazyObj) slog.Info("foo", "my lazy", lazyObj) ``` --- cmd/geth/logtestcmd_active.go | 10 ++-- cmd/geth/testdata/logging/logtest-json.txt | 3 +- cmd/geth/testdata/logging/logtest-logfmt.txt | 3 +- .../testdata/logging/logtest-terminal.txt | 5 +- internal/testlog/testlog.go | 6 +- log/format.go | 55 ++++--------------- log/handler.go | 39 +------------ log/logger.go | 28 +++------- log/logger_test.go | 4 +- p2p/discover/table.go | 7 ++- p2p/msgrate/msgrate.go | 5 +- 11 files changed, 49 insertions(+), 116 deletions(-) diff --git a/cmd/geth/logtestcmd_active.go b/cmd/geth/logtestcmd_active.go index 0ca4cc621d..5cce1ec6ab 100644 --- a/cmd/geth/logtestcmd_active.go +++ b/cmd/geth/logtestcmd_active.go @@ -43,6 +43,7 @@ This command is only meant for testing. type customQuotedStringer struct { } + func (c customQuotedStringer) String() string { return "output with 'quotes'" } @@ -80,8 +81,6 @@ func logTest(ctx *cli.Context) error { log.Info("uint64", "18,446,744,073,709,551,615", uint64(math.MaxUint64)) } { // Special characters - - log.Info("Special chars in value", "key", "special \r\n\t chars") log.Info("Special chars in key", "special \n\t chars", "value") @@ -103,9 +102,6 @@ func logTest(ctx *cli.Context) error { var c customQuotedStringer log.Info("a custom stringer that emits quoted text", "output", c) } - { // Lazy eval - log.Info("Lazy evaluation of value", "key", log.Lazy{Fn: func() interface{} { return "lazy value" }}) - } { // Multi-line message log.Info("A message with wonky \U0001F4A9 characters") log.Info("A multiline message \nINFO [10-18|14:11:31.106] with wonky characters \U0001F4A9") @@ -166,6 +162,10 @@ func logTest(ctx *cli.Context) error { { // Logging with 'reserved' keys log.Info("Using keys 't', 'lvl', 'time', 'level' and 'msg'", "t", "t", "time", "time", "lvl", "lvl", "level", "level", "msg", "msg") } + { // Logging with wrong attr-value pairs + log.Info("Odd pair (1 attr)", "key") + log.Info("Odd pair (3 attr)", "key", "value", "key2") + } return nil } diff --git a/cmd/geth/testdata/logging/logtest-json.txt b/cmd/geth/testdata/logging/logtest-json.txt index bdc1ae4de6..3bfe718660 100644 --- a/cmd/geth/testdata/logging/logtest-json.txt +++ b/cmd/geth/testdata/logging/logtest-json.txt @@ -21,7 +21,6 @@ {"t":"2023-11-22T15:42:00.408197+08:00","lvl":"info","msg":"an error message with quotes","error":"this is an 'error'"} {"t":"2023-11-22T15:42:00.408202+08:00","lvl":"info","msg":"Custom Stringer value","2562047h47m16.854s":"2562047h47m16.854s"} {"t":"2023-11-22T15:42:00.408208+08:00","lvl":"info","msg":"a custom stringer that emits quoted text","output":"output with 'quotes'"} -{"t":"2023-11-22T15:42:00.408215+08:00","lvl":"info","msg":"Lazy evaluation of value","key":"lazy value"} {"t":"2023-11-22T15:42:00.408219+08:00","lvl":"info","msg":"A message with wonky 💩 characters"} {"t":"2023-11-22T15:42:00.408222+08:00","lvl":"info","msg":"A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩"} {"t":"2023-11-22T15:42:00.408226+08:00","lvl":"info","msg":"A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above"} @@ -49,3 +48,5 @@ {"t":"2023-11-22T15:42:00.40835+08:00","lvl":"info","msg":"raw nil","res":null} {"t":"2023-11-22T15:42:00.408354+08:00","lvl":"info","msg":"(*uint64)(nil)","res":null} {"t":"2023-11-22T15:42:00.408361+08:00","lvl":"info","msg":"Using keys 't', 'lvl', 'time', 'level' and 'msg'","t":"t","time":"time","lvl":"lvl","level":"level","msg":"msg"} +{"t":"2023-11-29T15:13:00.195655931+01:00","lvl":"info","msg":"Odd pair (1 attr)","key":null,"LOG_ERROR":"Normalized odd number of arguments by adding nil"} +{"t":"2023-11-29T15:13:00.195681832+01:00","lvl":"info","msg":"Odd pair (3 attr)","key":"value","key2":null,"LOG_ERROR":"Normalized odd number of arguments by adding nil"} diff --git a/cmd/geth/testdata/logging/logtest-logfmt.txt b/cmd/geth/testdata/logging/logtest-logfmt.txt index 114569e467..f20d66635d 100644 --- a/cmd/geth/testdata/logging/logtest-logfmt.txt +++ b/cmd/geth/testdata/logging/logtest-logfmt.txt @@ -21,7 +21,6 @@ t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColor t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="an error message with quotes" error="this is an 'error'" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Custom Stringer value" 2562047h47m16.854s=2562047h47m16.854s t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="a custom stringer that emits quoted text" output="output with 'quotes'" -t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Lazy evaluation of value" key="lazy value" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A message with wonky 💩 characters" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above" @@ -49,3 +48,5 @@ t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nil-custom-struct res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="raw nil" res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(*uint64)(nil) res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Using keys 't', 'lvl', 'time', 'level' and 'msg'" t=t time=time lvl=lvl level=level msg=msg +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Odd pair (1 attr)" key= LOG_ERROR="Normalized odd number of arguments by adding nil" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Odd pair (3 attr)" key=value key2= LOG_ERROR="Normalized odd number of arguments by adding nil" diff --git a/cmd/geth/testdata/logging/logtest-terminal.txt b/cmd/geth/testdata/logging/logtest-terminal.txt index 4da3f49d46..e3b562117c 100644 --- a/cmd/geth/testdata/logging/logtest-terminal.txt +++ b/cmd/geth/testdata/logging/logtest-terminal.txt @@ -21,8 +21,7 @@ INFO [xx-xx|xx:xx:xx.xxx] "\x1b[35mColored\x1b[0m[" "\x1b[35mColo INFO [xx-xx|xx:xx:xx.xxx] an error message with quotes error="this is an 'error'" INFO [xx-xx|xx:xx:xx.xxx] Custom Stringer value 2562047h47m16.854s=2562047h47m16.854s INFO [xx-xx|xx:xx:xx.xxx] a custom stringer that emits quoted text output="output with 'quotes'" -INFO [xx-xx|xx:xx:xx.xxx] Lazy evaluation of value key="lazy value" -INFO [xx-xx|xx:xx:xx.xxx] "A message with wonky 💩 characters" +INFO [xx-xx|xx:xx:xx.xxx] "A message with wonky 💩 characters" INFO [xx-xx|xx:xx:xx.xxx] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" INFO [xx-xx|xx:xx:xx.xxx] A multiline message LALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above @@ -50,3 +49,5 @@ INFO [xx-xx|xx:xx:xx.xxx] nil-custom-struct res= INFO [xx-xx|xx:xx:xx.xxx] raw nil res= INFO [xx-xx|xx:xx:xx.xxx] (*uint64)(nil) res= INFO [xx-xx|xx:xx:xx.xxx] Using keys 't', 'lvl', 'time', 'level' and 'msg' t=t time=time lvl=lvl level=level msg=msg +INFO [xx-xx|xx:xx:xx.xxx] Odd pair (1 attr) key= LOG_ERROR="Normalized odd number of arguments by adding nil" +INFO [xx-xx|xx:xx:xx.xxx] Odd pair (3 attr) key=value key2= LOG_ERROR="Normalized odd number of arguments by adding nil" diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index a7899c8158..037b7ee9c1 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -100,6 +100,10 @@ func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} +func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { + return l.l.Enabled(ctx, level) +} + func (l *logger) Trace(msg string, ctx ...interface{}) { l.t.Helper() l.mu.Lock() @@ -183,7 +187,7 @@ func (h *bufHandler) terminalFormat(r slog.Record) string { } for _, attr := range attrs { - fmt.Fprintf(buf, " %s=%s", attr.Key, string(log.FormatSlogValue(attr.Value, true, nil))) + fmt.Fprintf(buf, " %s=%s", attr.Key, string(log.FormatSlogValue(attr.Value, nil))) } buf.WriteByte('\n') return buf.String() diff --git a/log/format.go b/log/format.go index a2bbcce9c0..6447f3c1f1 100644 --- a/log/format.go +++ b/log/format.go @@ -23,22 +23,6 @@ const ( // 40 spaces var spaces = []byte(" ") -type Format interface { - Format(r slog.Record) []byte -} - -// FormatFunc returns a new Format object which uses -// the given function to perform record formatting. -func FormatFunc(f func(slog.Record) []byte) Format { - return formatFunc(f) -} - -type formatFunc func(slog.Record) []byte - -func (f formatFunc) Format(r slog.Record) []byte { - return f(r) -} - // TerminalStringer is an analogous interface to the stdlib stringer, allowing // own types to have custom shortened serialization formats when printed to the // screen. @@ -46,7 +30,7 @@ type TerminalStringer interface { TerminalString() string } -func (h *TerminalHandler) TerminalFormat(buf []byte, r slog.Record, usecolor bool) []byte { +func (h *TerminalHandler) format(buf []byte, r slog.Record, usecolor bool) []byte { msg := escapeMessage(r.Message) var color = "" if usecolor { @@ -88,13 +72,13 @@ func (h *TerminalHandler) TerminalFormat(buf []byte, r slog.Record, usecolor boo if (r.NumAttrs()+len(h.attrs)) > 0 && length < termMsgJust { b.Write(spaces[:termMsgJust-length]) } - // print the keys logfmt style - h.logfmt(b, r, color) + // print the attributes + h.formatAttributes(b, r, color) return b.Bytes() } -func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) { +func (h *TerminalHandler) formatAttributes(buf *bytes.Buffer, r slog.Record, color string) { // tmp is a temporary buffer we use, until bytes.Buffer.AvailableBuffer() (1.21) // can be used. var tmp = make([]byte, 40) @@ -112,7 +96,7 @@ func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) buf.WriteByte('=') } //val := FormatSlogValue(attr.Value, true, buf.AvailableBuffer()) - val := FormatSlogValue(attr.Value, true, tmp[:0]) + val := FormatSlogValue(attr.Value, tmp[:0]) padding := h.fieldPadding[attr.Key] @@ -140,8 +124,8 @@ func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) buf.WriteByte('\n') } -// FormatSlogValue formats a slog.Value for serialization -func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { +// FormatSlogValue formats a slog.Value for serialization to terminal. +func FormatSlogValue(v slog.Value, tmp []byte) (result []byte) { var value any defer func() { if err := recover(); err != nil { @@ -156,11 +140,9 @@ func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { switch v.Kind() { case slog.KindString: return appendEscapeString(tmp, v.String()) - case slog.KindAny: - value = v.Any() - case slog.KindInt64: // All int-types (int8 ,int16 etc) wind up here + case slog.KindInt64: // All int-types (int8, int16 etc) wind up here return appendInt64(tmp, v.Int64()) - case slog.KindUint64: // All uint-types (int8 ,int16 etc) wind up here + case slog.KindUint64: // All uint-types (uint8, uint16 etc) wind up here return appendUint64(tmp, v.Uint64(), false) case slog.KindFloat64: return strconv.AppendFloat(tmp, v.Float64(), floatFormat, 3, 64) @@ -180,27 +162,14 @@ func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { return []byte("") } switch v := value.(type) { - case *big.Int: - // Big ints get consumed by the Stringer clause, so we need to handle - // them earlier on. - if v == nil { - return append(tmp, []byte("")...) - } + case *big.Int: // Need to be before fmt.Stringer-clause return appendBigInt(tmp, v) - - case *uint256.Int: - // Uint256s get consumed by the Stringer clause, so we need to handle - // them earlier on. - if v == nil { - return append(tmp, []byte("")...) - } + case *uint256.Int: // Need to be before fmt.Stringer-clause return appendU256(tmp, v) case error: return appendEscapeString(tmp, v.Error()) case TerminalStringer: - if term { - return appendEscapeString(tmp, v.TerminalString()) // Custom terminal stringer provided, use that - } + return appendEscapeString(tmp, v.TerminalString()) case fmt.Stringer: return appendEscapeString(tmp, v.String()) } diff --git a/log/handler.go b/log/handler.go index 1a25577450..7459aad891 100644 --- a/log/handler.go +++ b/log/handler.go @@ -13,42 +13,6 @@ import ( "golang.org/x/exp/slog" ) -// Lazy allows you to defer calculation of a logged value that is expensive -// to compute until it is certain that it must be evaluated with the given filters. -// -// You may wrap any function which takes no arguments to Lazy. It may return any -// number of values of any type. -type Lazy struct { - Fn interface{} -} - -func evaluateLazy(lz Lazy) (interface{}, error) { - t := reflect.TypeOf(lz.Fn) - - if t.Kind() != reflect.Func { - return nil, fmt.Errorf("INVALID_LAZY, not func: %+v", lz.Fn) - } - - if t.NumIn() > 0 { - return nil, fmt.Errorf("INVALID_LAZY, func takes args: %+v", lz.Fn) - } - - if t.NumOut() == 0 { - return nil, fmt.Errorf("INVALID_LAZY, no func return val: %+v", lz.Fn) - } - - value := reflect.ValueOf(lz.Fn) - results := value.Call([]reflect.Value{}) - if len(results) == 1 { - return results[0].Interface(), nil - } - values := make([]interface{}, len(results)) - for i, v := range results { - values[i] = v.Interface() - } - return values, nil -} - type discardHandler struct{} // DiscardHandler returns a no-op handler @@ -112,7 +76,7 @@ func NewTerminalHandlerWithLevel(wr io.Writer, lvl slog.Level, useColor bool) *T func (h *TerminalHandler) Handle(_ context.Context, r slog.Record) error { h.mu.Lock() defer h.mu.Unlock() - buf := h.TerminalFormat(h.buf, r, h.useColor) + buf := h.format(h.buf, r, h.useColor) h.wr.Write(buf) h.buf = buf[:0] return nil @@ -149,6 +113,7 @@ func (l *leveler) Level() slog.Level { return l.minLevel } +// JSONHandler returns a handler which prints records in JSON format. func JSONHandler(wr io.Writer) slog.Handler { return slog.NewJSONHandler(wr, &slog.HandlerOptions{ ReplaceAttr: builtinReplaceJSON, diff --git a/log/logger.go b/log/logger.go index 3e227745ad..93d62f080b 100644 --- a/log/logger.go +++ b/log/logger.go @@ -134,6 +134,9 @@ type Logger interface { // Write logs a message at the specified level Write(level slog.Level, msg string, attrs ...any) + + // Enabled reports whether l emits log records at the given context and level. + Enabled(ctx context.Context, level slog.Level) bool } type logger struct { @@ -159,26 +162,6 @@ func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if len(attrs)%2 != 0 { attrs = append(attrs, nil, errorKey, "Normalized odd number of arguments by adding nil") } - - // evaluate lazy values - var hadErr bool - for i := 1; i < len(attrs); i += 2 { - lz, ok := attrs[i].(Lazy) - if ok { - v, err := evaluateLazy(lz) - if err != nil { - hadErr = true - attrs[i] = err - } else { - attrs[i] = v - } - } - } - - if hadErr { - attrs = append(attrs, errorKey, "bad lazy") - } - r := slog.NewRecord(time.Now(), level, msg, pcs[0]) r.Add(attrs...) l.inner.Handler().Handle(context.Background(), r) @@ -196,6 +179,11 @@ func (l *logger) New(ctx ...interface{}) Logger { return l.With(ctx...) } +// Enabled reports whether l emits log records at the given context and level. +func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { + return l.inner.Enabled(ctx, level) +} + func (l *logger) Trace(msg string, ctx ...interface{}) { l.Write(LevelTrace, msg, ctx...) } diff --git a/log/logger_test.go b/log/logger_test.go index 27e90c5fd2..a633f5ad7a 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -107,7 +107,6 @@ func TestLoggerOutput(t *testing.T) { bigint = big.NewInt(100) nilbig *big.Int err = fmt.Errorf("Oh nooes it's crap") - lazy = Lazy{Fn: func() interface{} { return "lazy value" }} smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) @@ -126,13 +125,12 @@ func TestLoggerOutput(t *testing.T) { "struct", customA, "struct", customB, "ptrstruct", &customA, - "lazy", lazy, "smalluint", smallUint, "bigUint", bigUint) have := out.String() t.Logf("output %v", out.String()) - want := `INFO [11-07|19:14:33.821] This is a message foo=123 bytes="[0 0 0 0 0 0 0 0 0 0]" bonk="a string with text" time=0001-01-01T00:00:00+0000 bigint=100 nilbig= err="Oh nooes it's crap" struct="{A:Foo B:12}" struct="{A:Foo\nLinebreak B:122}" ptrstruct="&{A:Foo B:12}" lazy="lazy value" smalluint=500,000 bigUint=1,600,660,942,523,603,594,864,898,306,482,794,244,293,965,082,972,225,630,372,095 + want := `INFO [11-07|19:14:33.821] This is a message foo=123 bytes="[0 0 0 0 0 0 0 0 0 0]" bonk="a string with text" time=0001-01-01T00:00:00+0000 bigint=100 nilbig= err="Oh nooes it's crap" struct="{A:Foo B:12}" struct="{A:Foo\nLinebreak B:122}" ptrstruct="&{A:Foo B:12}" smalluint=500,000 bigUint=1,600,660,942,523,603,594,864,898,306,482,794,244,293,965,082,972,225,630,372,095 ` if !bytes.Equal([]byte(have)[25:], []byte(want)[25:]) { t.Errorf("Error\nhave: %q\nwant: %q", have, want) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index f476d2079f..e6dafb0dca 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -23,6 +23,7 @@ package discover import ( + "context" crand "crypto/rand" "encoding/binary" "fmt" @@ -330,8 +331,10 @@ func (tab *Table) loadSeedNodes() { seeds = append(seeds, tab.nursery...) for i := range seeds { seed := seeds[i] - age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) }} - tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) + if tab.log.Enabled(context.Background(), log.LevelTrace) { + age := time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) + tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) + } tab.addSeenNode(seed) } } diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 4f08792242..de1a3177db 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -18,6 +18,7 @@ package msgrate import ( + "context" "errors" "math" "sort" @@ -410,7 +411,9 @@ func (t *Trackers) tune() { t.tuned = time.Now() t.log.Debug("Recalculated msgrate QoS values", "rtt", t.roundtrip, "confidence", t.confidence, "ttl", t.targetTimeout(), "next", t.tuned.Add(t.roundtrip)) - t.log.Trace("Debug dump of mean capacities", "caps", log.Lazy{Fn: t.meanCapacities}) + if t.log.Enabled(context.Background(), log.LevelTrace) { + t.log.Trace("Debug dump of mean capacities", "caps", t.meanCapacities()) + } } // detune reduces the tracker's confidence in order to make fresh measurements From 69576df2544d9a6c59c5659b82a064edc9845874 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 5 Dec 2023 14:45:40 +0100 Subject: [PATCH 07/15] .github: use github actions to run 32-bit linux tests (#28549) use github actions to run 32-bit linux tests --- .github/workflows/go.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000000..7924c521e8 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,23 @@ +name: i386 linux tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.21.4 + - name: Run tests + run: go test ./... + env: + GOOS: linux + GOARCH: 386 From 55b483d82aa23772f8a8b330e3837a07c4fd00df Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 6 Dec 2023 11:41:04 +0100 Subject: [PATCH 08/15] ethdb/pebble: remove a dependency (#28627) The dependency was not really used anyway, so we can get rid of it. Co-authored-by: Felix Lange --- ethdb/pebble/pebble.go | 3 +-- go.mod | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index d58329c6d6..af4686cf5b 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -25,7 +25,6 @@ import ( "sync/atomic" "time" - "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble" "github.com/cockroachdb/pebble/bloom" "github.com/ethereum/go-ethereum/common" @@ -131,7 +130,7 @@ func (l panicLogger) Errorf(format string, args ...interface{}) { } func (l panicLogger) Fatalf(format string, args ...interface{}) { - panic(errors.Errorf("fatal: "+format, args...)) + panic(fmt.Errorf("fatal: "+format, args...)) } // New returns a wrapped pebble DB object. The namespace is the prefix that the diff --git a/go.mod b/go.mod index 75c2b899d6..8f99a00754 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/errors v1.8.1 github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 @@ -91,6 +90,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/errors v1.8.1 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect From a113497dd7e5f5116996b34c514c1c9273bfcc97 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:07:20 +0100 Subject: [PATCH 09/15] tests/fuzzers/bls12381: deactivate BLS fuzzer when CGO_ENABLED=0 (#28653) tests/fuzzers/bls12381: deactivate fuzzer when CGO_ENABLED=0 --- tests/fuzzers/bls12381/bls12381_fuzz.go | 3 +++ tests/fuzzers/bls12381/bls12381_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index f04524f76a..9a5c566540 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -14,6 +14,9 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build cgo +// +build cgo + package bls import ( diff --git a/tests/fuzzers/bls12381/bls12381_test.go b/tests/fuzzers/bls12381/bls12381_test.go index 59e4db31d5..3e88979d16 100644 --- a/tests/fuzzers/bls12381/bls12381_test.go +++ b/tests/fuzzers/bls12381/bls12381_test.go @@ -14,6 +14,9 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build cgo +// +build cgo + package bls import "testing" From 77c4bbcaa5f554f4cd73bdb7033d17b1fec493e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Thu, 7 Dec 2023 11:45:09 +0100 Subject: [PATCH 10/15] build: upgrade -dlgo version to Go 1.21.5 (#28648) --- build/checksums.txt | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index c96bd85667..8d735fdb3d 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,22 +5,22 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/ 485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1 fixtures_develop.tar.gz -# version:golang 1.21.4 +# version:golang 1.21.5 # https://go.dev/dl/ -47b26a83d2b65a3c1c1bcace273b69bee49a7a7b5168a7604ded3d26a37bd787 go1.21.4.src.tar.gz -cd3bdcc802b759b70e8418bc7afbc4a65ca73a3fe576060af9fc8a2a5e71c3b8 go1.21.4.darwin-amd64.tar.gz -8b7caf2ac60bdff457dba7d4ff2a01def889592b834453431ae3caecf884f6a5 go1.21.4.darwin-arm64.tar.gz -f1e685d086eb36f4be5b8b953b52baf7752bc6235400d84bb7d87e500b65f03e go1.21.4.freebsd-386.tar.gz -59f9b32187efb98d344a3818a631d3815ebb5c7bbefc367bab6515caaca544e9 go1.21.4.freebsd-amd64.tar.gz -64d3e5d295806e137c9e39d1e1f10b00a30fcd5c2f230d72b3298f579bb3c89a go1.21.4.linux-386.tar.gz -73cac0215254d0c7d1241fa40837851f3b9a8a742d0b54714cbdfb3feaf8f0af go1.21.4.linux-amd64.tar.gz -ce1983a7289856c3a918e1fd26d41e072cc39f928adfb11ba1896440849b95da go1.21.4.linux-arm64.tar.gz -6c62e89113750cc77c498194d13a03fadfda22bd2c7d44e8a826fd354db60252 go1.21.4.linux-armv6l.tar.gz -2c63b36d2adcfb22013102a2ee730f058ec2f93b9f27479793c80b2e3641783f go1.21.4.linux-ppc64le.tar.gz -7a75ba4afc7a96058ca65903d994cd862381825d7dca12b2183f087c757c26c0 go1.21.4.linux-s390x.tar.gz -870a0e462b94671dc2d6cac707e9e19f7524fdc3c90711e6cd4450c3713a8ce0 go1.21.4.windows-386.zip -79e5428e068c912d9cfa6cd115c13549856ec689c1332eac17f5d6122e19d595 go1.21.4.windows-amd64.zip -58bc7c6f4d4c72da2df4d2650c8222fe03c9978070eb3c66be8bbaa2a4757ac1 go1.21.4.windows-arm64.zip +285cbbdf4b6e6e62ed58f370f3f6d8c30825d6e56c5853c66d3c23bcdb09db19 go1.21.5.src.tar.gz +a2e1d5743e896e5fe1e7d96479c0a769254aed18cf216cf8f4c3a2300a9b3923 go1.21.5.darwin-amd64.tar.gz +d0f8ac0c4fb3efc223a833010901d02954e3923cfe2c9a2ff0e4254a777cc9cc go1.21.5.darwin-arm64.tar.gz +2c05bbe0dc62456b90b7ddd354a54f373b7c377a98f8b22f52ab694b4f6cca58 go1.21.5.freebsd-386.tar.gz +30b6c64e9a77129605bc12f836422bf09eec577a8c899ee46130aeff81567003 go1.21.5.freebsd-amd64.tar.gz +8f4dba9cf5c61757bbd7e9ebdb93b6a30a1b03f4a636a1ba0cc2f27b907ab8e1 go1.21.5.linux-386.tar.gz +e2bc0b3e4b64111ec117295c088bde5f00eeed1567999ff77bc859d7df70078e go1.21.5.linux-amd64.tar.gz +841cced7ecda9b2014f139f5bab5ae31785f35399f236b8b3e75dff2a2978d96 go1.21.5.linux-arm64.tar.gz +837f4bf4e22fcdf920ffeaa4abf3d02d1314e03725431065f4d44c46a01b42fe go1.21.5.linux-armv6l.tar.gz +907b8c6ec4be9b184952e5d3493be66b1746442394a8bc78556c56834cd7c38b go1.21.5.linux-ppc64le.tar.gz +9c4a81b72ebe44368813cd03684e1080a818bf915d84163abae2ed325a1b2dc0 go1.21.5.linux-s390x.tar.gz +6da2418889dfb37763d0eb149c4a8d728c029e12f0cd54fbca0a31ae547e2d34 go1.21.5.windows-386.zip +bbe603cde7c9dee658f45164b4d06de1eff6e6e6b800100824e7c00d56a9a92f go1.21.5.windows-amd64.zip +9b7acca50e674294e43202df4fbc26d5af4d8bc3170a3342a1514f09a2dab5e9 go1.21.5.windows-arm64.zip # version:golangci 1.51.1 # https://github.com/golangci/golangci-lint/releases/ From 5a45e7a631e0a2457178f346a03c4128efb7d009 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Fri, 8 Dec 2023 09:40:50 +0100 Subject: [PATCH 11/15] =?UTF-8?q?rpc:=20fix=20ns/=C2=B5s=20mismatch=20in?= =?UTF-8?q?=20metrics=20(#28649)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rpc/duration/all meter was in nanoseconds, the individual meter in microseconds. This PR changes it so both of them use nanoseconds. --- metrics/timer.go | 10 ++++------ rpc/metrics.go | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/metrics/timer.go b/metrics/timer.go index 576ad8aa3e..bb8def82fb 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -106,20 +106,18 @@ func (t *StandardTimer) Time(f func()) { t.Update(time.Since(ts)) } -// Record the duration of an event. +// Record the duration of an event, in nanoseconds. func (t *StandardTimer) Update(d time.Duration) { t.mutex.Lock() defer t.mutex.Unlock() - t.histogram.Update(int64(d)) + t.histogram.Update(d.Nanoseconds()) t.meter.Mark(1) } // Record the duration of an event that started at a time and ends now. +// The record uses nanoseconds. func (t *StandardTimer) UpdateSince(ts time.Time) { - t.mutex.Lock() - defer t.mutex.Unlock() - t.histogram.Update(int64(time.Since(ts))) - t.meter.Mark(1) + t.Update(time.Since(ts)) } // timerSnapshot is a read-only copy of another Timer. diff --git a/rpc/metrics.go b/rpc/metrics.go index b1f1284535..ef7449ce05 100644 --- a/rpc/metrics.go +++ b/rpc/metrics.go @@ -46,5 +46,5 @@ func updateServeTimeHistogram(method string, success bool, elapsed time.Duration metrics.NewExpDecaySample(1028, 0.015), ) } - metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(elapsed.Microseconds()) + metrics.GetOrRegisterHistogramLazy(h, nil, sampler).Update(elapsed.Nanoseconds()) } From 1048e2d6a36b8c3fb467d60fd33122bae0cc7ebe Mon Sep 17 00:00:00 2001 From: Martin HS Date: Fri, 8 Dec 2023 11:06:01 +0100 Subject: [PATCH 12/15] cmd/evm: fix dump after state-test exec (#28650) The dump after state-test didn't work, the problem was an error, "Already committed", which was silently ignored. This change re-initialises the state, so the dumping works again. --- cmd/evm/staterunner.go | 17 +++++++++-------- core/state/dump.go | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index 618ddf2ede..6e751b630f 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -100,18 +100,19 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) { - if state != nil { - root := state.IntermediateRoot(false) + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, statedb *state.StateDB) { + var root common.Hash + if statedb != nil { + root = statedb.IntermediateRoot(false) result.Root = &root if jsonOut { fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) } - } - // Dump any state to aid debugging - if dump { - dump := state.RawDump(nil) - result.State = &dump + if dump { // Dump any state to aid debugging + cpy, _ := state.New(root, statedb.Database(), nil) + dump := cpy.RawDump(nil) + result.State = &dump + } } if err != nil { // Test failed, mark as so diff --git a/core/state/dump.go b/core/state/dump.go index cf46621144..55abb50f1c 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -129,6 +129,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] trieIt, err := s.trie.NodeIterator(conf.Start) if err != nil { + log.Error("Trie dumping error", "err", err) return nil } it := trie.NewIterator(trieIt) From fff843cfafddb43aebad6e3e5ad8b0f68143759d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 8 Dec 2023 13:38:00 +0100 Subject: [PATCH 13/15] beacon/light: add CommitteeChain (#27766) This change implements CommitteeChain which is a key component of the beacon light client. It is a passive data structure that can validate, hold and update a chain of beacon light sync committees and updates, starting from a checkpoint that proves the starting committee through a beacon block hash, header and corresponding state. Once synced to the current sync period, CommitteeChain can also validate signed beacon headers. --- beacon/light/canonical.go | 125 ++++++ beacon/light/committee_chain.go | 514 ++++++++++++++++++++++ beacon/light/committee_chain_test.go | 356 +++++++++++++++ beacon/light/range.go | 78 ++++ beacon/light/test_helpers.go | 152 +++++++ beacon/types/{update.go => light_sync.go} | 18 + core/rawdb/schema.go | 4 + 7 files changed, 1247 insertions(+) create mode 100644 beacon/light/canonical.go create mode 100644 beacon/light/committee_chain.go create mode 100644 beacon/light/committee_chain_test.go create mode 100644 beacon/light/range.go create mode 100644 beacon/light/test_helpers.go rename beacon/types/{update.go => light_sync.go} (88%) diff --git a/beacon/light/canonical.go b/beacon/light/canonical.go new file mode 100644 index 0000000000..b5371493b4 --- /dev/null +++ b/beacon/light/canonical.go @@ -0,0 +1,125 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "encoding/binary" + "fmt" + + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" +) + +// canonicalStore stores instances of the given type in a database and caches +// them in memory, associated with a continuous range of period numbers. +// Note: canonicalStore is not thread safe and it is the caller's responsibility +// to avoid concurrent access. +type canonicalStore[T any] struct { + keyPrefix []byte + periods periodRange + cache *lru.Cache[uint64, T] +} + +// newCanonicalStore creates a new canonicalStore and loads all keys associated +// with the keyPrefix in order to determine the ranges available in the database. +func newCanonicalStore[T any](db ethdb.Iteratee, keyPrefix []byte) (*canonicalStore[T], error) { + cs := &canonicalStore[T]{ + keyPrefix: keyPrefix, + cache: lru.NewCache[uint64, T](100), + } + var ( + iter = db.NewIterator(keyPrefix, nil) + kl = len(keyPrefix) + first = true + ) + defer iter.Release() + + for iter.Next() { + if len(iter.Key()) != kl+8 { + log.Warn("Invalid key length in the canonical chain database", "key", fmt.Sprintf("%#x", iter.Key())) + continue + } + period := binary.BigEndian.Uint64(iter.Key()[kl : kl+8]) + if first { + cs.periods.Start = period + } else if cs.periods.End != period { + return nil, fmt.Errorf("gap in the canonical chain database between periods %d and %d", cs.periods.End, period-1) + } + first = false + cs.periods.End = period + 1 + } + return cs, nil +} + +// databaseKey returns the database key belonging to the given period. +func (cs *canonicalStore[T]) databaseKey(period uint64) []byte { + return binary.BigEndian.AppendUint64(append([]byte{}, cs.keyPrefix...), period) +} + +// add adds the given item to the database. It also ensures that the range remains +// continuous. Can be used either with a batch or database backend. +func (cs *canonicalStore[T]) add(backend ethdb.KeyValueWriter, period uint64, value T) error { + if !cs.periods.canExpand(period) { + return fmt.Errorf("period expansion is not allowed, first: %d, next: %d, period: %d", cs.periods.Start, cs.periods.End, period) + } + enc, err := rlp.EncodeToBytes(value) + if err != nil { + return err + } + if err := backend.Put(cs.databaseKey(period), enc); err != nil { + return err + } + cs.cache.Add(period, value) + cs.periods.expand(period) + return nil +} + +// deleteFrom removes items starting from the given period. +func (cs *canonicalStore[T]) deleteFrom(db ethdb.KeyValueWriter, fromPeriod uint64) (deleted periodRange) { + keepRange, deleteRange := cs.periods.split(fromPeriod) + deleteRange.each(func(period uint64) { + db.Delete(cs.databaseKey(period)) + cs.cache.Remove(period) + }) + cs.periods = keepRange + return deleteRange +} + +// get returns the item at the given period or the null value of the given type +// if no item is present. +func (cs *canonicalStore[T]) get(backend ethdb.KeyValueReader, period uint64) (T, bool) { + var null, value T + if !cs.periods.contains(period) { + return null, false + } + if value, ok := cs.cache.Get(period); ok { + return value, true + } + enc, err := backend.Get(cs.databaseKey(period)) + if err != nil { + log.Error("Canonical store value not found", "period", period, "start", cs.periods.Start, "end", cs.periods.End) + return null, false + } + if err := rlp.DecodeBytes(enc, &value); err != nil { + log.Error("Error decoding canonical store value", "error", err) + return null, false + } + cs.cache.Add(period, value) + return value, true +} diff --git a/beacon/light/committee_chain.go b/beacon/light/committee_chain.go new file mode 100644 index 0000000000..d707f8cc34 --- /dev/null +++ b/beacon/light/committee_chain.go @@ -0,0 +1,514 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "errors" + "fmt" + "math" + "sync" + "time" + + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" +) + +var ( + ErrNeedCommittee = errors.New("sync committee required") + ErrInvalidUpdate = errors.New("invalid committee update") + ErrInvalidPeriod = errors.New("invalid update period") + ErrWrongCommitteeRoot = errors.New("wrong committee root") + ErrCannotReorg = errors.New("can not reorg committee chain") +) + +// CommitteeChain is a passive data structure that can validate, hold and update +// a chain of beacon light sync committees and updates. It requires at least one +// externally set fixed committee root at the beginning of the chain which can +// be set either based on a BootstrapData or a trusted source (a local beacon +// full node). This makes the structure useful for both light client and light +// server setups. +// +// It always maintains the following consistency constraints: +// - a committee can only be present if its root hash matches an existing fixed +// root or if it is proven by an update at the previous period +// - an update can only be present if a committee is present at the same period +// and the update signature is valid and has enough participants. +// The committee at the next period (proven by the update) should also be +// present (note that this means they can only be added together if neither +// is present yet). If a fixed root is present at the next period then the +// update can only be present if it proves the same committee root. +// +// Once synced to the current sync period, CommitteeChain can also validate +// signed beacon headers. +type CommitteeChain struct { + // chainmu guards against concurrent access to the canonicalStore structures + // (updates, committees, fixedCommitteeRoots) and ensures that they stay consistent + // with each other and with committeeCache. + chainmu sync.RWMutex + db ethdb.KeyValueStore + updates *canonicalStore[*types.LightClientUpdate] + committees *canonicalStore[*types.SerializedSyncCommittee] + fixedCommitteeRoots *canonicalStore[common.Hash] + committeeCache *lru.Cache[uint64, syncCommittee] // cache deserialized committees + + clock mclock.Clock // monotonic clock (simulated clock in tests) + unixNano func() int64 // system clock (simulated clock in tests) + sigVerifier committeeSigVerifier // BLS sig verifier (dummy verifier in tests) + + config *types.ChainConfig + signerThreshold int + minimumUpdateScore types.UpdateScore + enforceTime bool // enforceTime specifies whether the age of a signed header should be checked +} + +// NewCommitteeChain creates a new CommitteeChain. +func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool) *CommitteeChain { + return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() }) +} + +// newCommitteeChain creates a new CommitteeChain with the option of replacing the +// clock source and signature verification for testing purposes. +func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain { + s := &CommitteeChain{ + committeeCache: lru.NewCache[uint64, syncCommittee](10), + db: db, + sigVerifier: sigVerifier, + clock: clock, + unixNano: unixNano, + config: config, + signerThreshold: signerThreshold, + enforceTime: enforceTime, + minimumUpdateScore: types.UpdateScore{ + SignerCount: uint32(signerThreshold), + SubPeriodIndex: params.SyncPeriodLength / 16, + }, + } + + var err1, err2, err3 error + if s.fixedCommitteeRoots, err1 = newCanonicalStore[common.Hash](db, rawdb.FixedCommitteeRootKey); err1 != nil { + log.Error("Error creating fixed committee root store", "error", err1) + } + if s.committees, err2 = newCanonicalStore[*types.SerializedSyncCommittee](db, rawdb.SyncCommitteeKey); err2 != nil { + log.Error("Error creating committee store", "error", err2) + } + if s.updates, err3 = newCanonicalStore[*types.LightClientUpdate](db, rawdb.BestUpdateKey); err3 != nil { + log.Error("Error creating update store", "error", err3) + } + if err1 != nil || err2 != nil || err3 != nil || !s.checkConstraints() { + log.Info("Resetting invalid committee chain") + s.Reset() + } + // roll back invalid updates (might be necessary if forks have been changed since last time) + for !s.updates.periods.isEmpty() { + update, ok := s.updates.get(s.db, s.updates.periods.End-1) + if !ok { + log.Error("Sync committee update missing", "period", s.updates.periods.End-1) + s.Reset() + break + } + if valid, err := s.verifyUpdate(update); err != nil { + log.Error("Error validating update", "period", s.updates.periods.End-1, "error", err) + } else if valid { + break + } + if err := s.rollback(s.updates.periods.End); err != nil { + log.Error("Error writing batch into chain database", "error", err) + } + } + if !s.committees.periods.isEmpty() { + log.Trace("Sync committee chain loaded", "first period", s.committees.periods.Start, "last period", s.committees.periods.End-1) + } + return s +} + +// checkConstraints checks committee chain validity constraints +func (s *CommitteeChain) checkConstraints() bool { + isNotInFixedCommitteeRootRange := func(r periodRange) bool { + return s.fixedCommitteeRoots.periods.isEmpty() || + r.Start < s.fixedCommitteeRoots.periods.Start || + r.Start >= s.fixedCommitteeRoots.periods.End + } + + valid := true + if !s.updates.periods.isEmpty() { + if isNotInFixedCommitteeRootRange(s.updates.periods) { + log.Error("Start update is not in the fixed roots range") + valid = false + } + if s.committees.periods.Start > s.updates.periods.Start || s.committees.periods.End <= s.updates.periods.End { + log.Error("Missing committees in update range") + valid = false + } + } + if !s.committees.periods.isEmpty() { + if isNotInFixedCommitteeRootRange(s.committees.periods) { + log.Error("Start committee is not in the fixed roots range") + valid = false + } + if s.committees.periods.End > s.fixedCommitteeRoots.periods.End && s.committees.periods.End > s.updates.periods.End+1 { + log.Error("Last committee is neither in the fixed roots range nor proven by updates") + valid = false + } + } + return valid +} + +// Reset resets the committee chain. +func (s *CommitteeChain) Reset() { + s.chainmu.Lock() + defer s.chainmu.Unlock() + + if err := s.rollback(0); err != nil { + log.Error("Error writing batch into chain database", "error", err) + } +} + +// CheckpointInit initializes a CommitteeChain based on the checkpoint. +// Note: if the chain is already initialized and the committees proven by the +// checkpoint do match the existing chain then the chain is retained and the +// new checkpoint becomes fixed. +func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error { + s.chainmu.Lock() + defer s.chainmu.Unlock() + + if err := bootstrap.Validate(); err != nil { + return err + } + + period := bootstrap.Header.SyncPeriod() + if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil { + s.Reset() + return err + } + if s.addFixedCommitteeRoot(period, bootstrap.CommitteeRoot) != nil { + s.Reset() + if err := s.addFixedCommitteeRoot(period, bootstrap.CommitteeRoot); err != nil { + s.Reset() + return err + } + } + if err := s.addFixedCommitteeRoot(period+1, common.Hash(bootstrap.CommitteeBranch[0])); err != nil { + s.Reset() + return err + } + if err := s.addCommittee(period, bootstrap.Committee); err != nil { + s.Reset() + return err + } + return nil +} + +// addFixedCommitteeRoot sets a fixed committee root at the given period. +// Note that the period where the first committee is added has to have a fixed +// root which can either come from a BootstrapData or a trusted source. +func (s *CommitteeChain) addFixedCommitteeRoot(period uint64, root common.Hash) error { + if root == (common.Hash{}) { + return ErrWrongCommitteeRoot + } + + batch := s.db.NewBatch() + oldRoot := s.getCommitteeRoot(period) + if !s.fixedCommitteeRoots.periods.canExpand(period) { + // Note: the fixed committee root range should always be continuous and + // therefore the expected syncing method is to forward sync and optionally + // backward sync periods one by one, starting from a checkpoint. The only + // case when a root that is not adjacent to the already fixed ones can be + // fixed is when the same root has already been proven by an update chain. + // In this case the all roots in between can and should be fixed. + // This scenario makes sense when a new trusted checkpoint is added to an + // existing chain, ensuring that it will not be rolled back (might be + // important in case of low signer participation rate). + if root != oldRoot { + return ErrInvalidPeriod + } + // if the old root exists and matches the new one then it is guaranteed + // that the given period is after the existing fixed range and the roots + // in between can also be fixed. + for p := s.fixedCommitteeRoots.periods.End; p < period; p++ { + if err := s.fixedCommitteeRoots.add(batch, p, s.getCommitteeRoot(p)); err != nil { + return err + } + } + } + if oldRoot != (common.Hash{}) && (oldRoot != root) { + // existing old root was different, we have to reorg the chain + if err := s.rollback(period); err != nil { + return err + } + } + if err := s.fixedCommitteeRoots.add(batch, period, root); err != nil { + return err + } + if err := batch.Write(); err != nil { + log.Error("Error writing batch into chain database", "error", err) + return err + } + return nil +} + +// deleteFixedCommitteeRootsFrom deletes fixed roots starting from the given period. +// It also maintains chain consistency, meaning that it also deletes updates and +// committees if they are no longer supported by a valid update chain. +func (s *CommitteeChain) deleteFixedCommitteeRootsFrom(period uint64) error { + if period >= s.fixedCommitteeRoots.periods.End { + return nil + } + batch := s.db.NewBatch() + s.fixedCommitteeRoots.deleteFrom(batch, period) + if s.updates.periods.isEmpty() || period <= s.updates.periods.Start { + // Note: the first period of the update chain should always be fixed so if + // the fixed root at the first update is removed then the entire update chain + // and the proven committees have to be removed. Earlier committees in the + // remaining fixed root range can stay. + s.updates.deleteFrom(batch, period) + s.deleteCommitteesFrom(batch, period) + } else { + // The update chain stays intact, some previously fixed committee roots might + // get unfixed but are still proven by the update chain. If there were + // committees present after the range proven by updates, those should be + // removed if the belonging fixed roots are also removed. + fromPeriod := s.updates.periods.End + 1 // not proven by updates + if period > fromPeriod { + fromPeriod = period // also not justified by fixed roots + } + s.deleteCommitteesFrom(batch, fromPeriod) + } + if err := batch.Write(); err != nil { + log.Error("Error writing batch into chain database", "error", err) + return err + } + return nil +} + +// deleteCommitteesFrom deletes committees starting from the given period. +func (s *CommitteeChain) deleteCommitteesFrom(batch ethdb.Batch, period uint64) { + deleted := s.committees.deleteFrom(batch, period) + for period := deleted.Start; period < deleted.End; period++ { + s.committeeCache.Remove(period) + } +} + +// addCommittee adds a committee at the given period if possible. +func (s *CommitteeChain) addCommittee(period uint64, committee *types.SerializedSyncCommittee) error { + if !s.committees.periods.canExpand(period) { + return ErrInvalidPeriod + } + root := s.getCommitteeRoot(period) + if root == (common.Hash{}) { + return ErrInvalidPeriod + } + if root != committee.Root() { + return ErrWrongCommitteeRoot + } + if !s.committees.periods.contains(period) { + if err := s.committees.add(s.db, period, committee); err != nil { + return err + } + s.committeeCache.Remove(period) + } + return nil +} + +// InsertUpdate adds a new update if possible. +func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error { + s.chainmu.Lock() + defer s.chainmu.Unlock() + + period := update.AttestedHeader.Header.SyncPeriod() + if !s.updates.periods.canExpand(period) || !s.committees.periods.contains(period) { + return ErrInvalidPeriod + } + if s.minimumUpdateScore.BetterThan(update.Score()) { + return ErrInvalidUpdate + } + oldRoot := s.getCommitteeRoot(period + 1) + reorg := oldRoot != (common.Hash{}) && oldRoot != update.NextSyncCommitteeRoot + if oldUpdate, ok := s.updates.get(s.db, period); ok && !update.Score().BetterThan(oldUpdate.Score()) { + // a better or equal update already exists; no changes, only fail if new one tried to reorg + if reorg { + return ErrCannotReorg + } + return nil + } + if s.fixedCommitteeRoots.periods.contains(period+1) && reorg { + return ErrCannotReorg + } + if ok, err := s.verifyUpdate(update); err != nil { + return err + } else if !ok { + return ErrInvalidUpdate + } + addCommittee := !s.committees.periods.contains(period+1) || reorg + if addCommittee { + if nextCommittee == nil { + return ErrNeedCommittee + } + if nextCommittee.Root() != update.NextSyncCommitteeRoot { + return ErrWrongCommitteeRoot + } + } + if reorg { + if err := s.rollback(period + 1); err != nil { + return err + } + } + batch := s.db.NewBatch() + if addCommittee { + if err := s.committees.add(batch, period+1, nextCommittee); err != nil { + return err + } + s.committeeCache.Remove(period + 1) + } + if err := s.updates.add(batch, period, update); err != nil { + return err + } + if err := batch.Write(); err != nil { + log.Error("Error writing batch into chain database", "error", err) + return err + } + log.Info("Inserted new committee update", "period", period, "next committee root", update.NextSyncCommitteeRoot) + return nil +} + +// NextSyncPeriod returns the next period where an update can be added and also +// whether the chain is initialized at all. +func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) { + s.chainmu.RLock() + defer s.chainmu.RUnlock() + + if s.committees.periods.isEmpty() { + return 0, false + } + if !s.updates.periods.isEmpty() { + return s.updates.periods.End, true + } + return s.committees.periods.End - 1, true +} + +// rollback removes all committees and fixed roots from the given period and updates +// starting from the previous period. +func (s *CommitteeChain) rollback(period uint64) error { + max := s.updates.periods.End + 1 + if s.committees.periods.End > max { + max = s.committees.periods.End + } + if s.fixedCommitteeRoots.periods.End > max { + max = s.fixedCommitteeRoots.periods.End + } + for max > period { + max-- + batch := s.db.NewBatch() + s.deleteCommitteesFrom(batch, max) + s.fixedCommitteeRoots.deleteFrom(batch, max) + if max > 0 { + s.updates.deleteFrom(batch, max-1) + } + if err := batch.Write(); err != nil { + log.Error("Error writing batch into chain database", "error", err) + return err + } + } + return nil +} + +// getCommitteeRoot returns the committee root at the given period, either fixed, +// proven by a previous update or both. It returns an empty hash if the committee +// root is unknown. +func (s *CommitteeChain) getCommitteeRoot(period uint64) common.Hash { + if root, ok := s.fixedCommitteeRoots.get(s.db, period); ok || period == 0 { + return root + } + if update, ok := s.updates.get(s.db, period-1); ok { + return update.NextSyncCommitteeRoot + } + return common.Hash{} +} + +// getSyncCommittee returns the deserialized sync committee at the given period. +func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error) { + if c, ok := s.committeeCache.Get(period); ok { + return c, nil + } + if sc, ok := s.committees.get(s.db, period); ok { + c, err := s.sigVerifier.deserializeSyncCommittee(sc) + if err != nil { + return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err) + } + s.committeeCache.Add(period, c) + return c, nil + } + return nil, fmt.Errorf("Missing serialized sync committee #%d", period) +} + +// VerifySignedHeader returns true if the given signed header has a valid signature +// according to the local committee chain. The caller should ensure that the +// committees advertised by the same source where the signed header came from are +// synced before verifying the signature. +// The age of the header is also returned (the time elapsed since the beginning +// of the given slot, according to the local system clock). If enforceTime is +// true then negative age (future) headers are rejected. +func (s *CommitteeChain) VerifySignedHeader(head types.SignedHeader) (bool, time.Duration, error) { + s.chainmu.RLock() + defer s.chainmu.RUnlock() + + return s.verifySignedHeader(head) +} + +func (s *CommitteeChain) verifySignedHeader(head types.SignedHeader) (bool, time.Duration, error) { + var age time.Duration + now := s.unixNano() + if head.Header.Slot < (uint64(now-math.MinInt64)/uint64(time.Second)-s.config.GenesisTime)/12 { + age = time.Duration(now - int64(time.Second)*int64(s.config.GenesisTime+head.Header.Slot*12)) + } else { + age = time.Duration(math.MinInt64) + } + if s.enforceTime && age < 0 { + return false, age, nil + } + committee, err := s.getSyncCommittee(types.SyncPeriod(head.SignatureSlot)) + if err != nil { + return false, 0, err + } + if committee == nil { + return false, age, nil + } + if signingRoot, err := s.config.Forks.SigningRoot(head.Header); err == nil { + return s.sigVerifier.verifySignature(committee, signingRoot, &head.Signature), age, nil + } + return false, age, nil +} + +// verifyUpdate checks whether the header signature is correct and the update +// fits into the specified constraints (assumes that the update has been +// successfully validated previously) +func (s *CommitteeChain) verifyUpdate(update *types.LightClientUpdate) (bool, error) { + // Note: SignatureSlot determines the sync period of the committee used for signature + // verification. Though in reality SignatureSlot is always bigger than update.Header.Slot, + // setting them as equal here enforces the rule that they have to be in the same sync + // period in order for the light client update proof to be meaningful. + ok, age, err := s.verifySignedHeader(update.AttestedHeader) + if age < 0 { + log.Warn("Future committee update received", "age", age) + } + return ok, err +} diff --git a/beacon/light/committee_chain_test.go b/beacon/light/committee_chain_test.go new file mode 100644 index 0000000000..60ea2a0efd --- /dev/null +++ b/beacon/light/committee_chain_test.go @@ -0,0 +1,356 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "crypto/rand" + "testing" + "time" + + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/ethdb/memorydb" +) + +var ( + testGenesis = newTestGenesis() + testGenesis2 = newTestGenesis() + + tfBase = newTestForks(testGenesis, types.Forks{ + &types.Fork{Epoch: 0, Version: []byte{0}}, + }) + tfAlternative = newTestForks(testGenesis, types.Forks{ + &types.Fork{Epoch: 0, Version: []byte{0}}, + &types.Fork{Epoch: 0x700, Version: []byte{1}}, + }) + tfAnotherGenesis = newTestForks(testGenesis2, types.Forks{ + &types.Fork{Epoch: 0, Version: []byte{0}}, + }) + + tcBase = newTestCommitteeChain(nil, tfBase, true, 0, 10, 400, false) + tcBaseWithInvalidUpdates = newTestCommitteeChain(tcBase, tfBase, false, 5, 10, 200, false) // signer count too low + tcBaseWithBetterUpdates = newTestCommitteeChain(tcBase, tfBase, false, 5, 10, 440, false) + tcReorgWithWorseUpdates = newTestCommitteeChain(tcBase, tfBase, true, 5, 10, 400, false) + tcReorgWithWorseUpdates2 = newTestCommitteeChain(tcBase, tfBase, true, 5, 10, 380, false) + tcReorgWithBetterUpdates = newTestCommitteeChain(tcBase, tfBase, true, 5, 10, 420, false) + tcReorgWithFinalizedUpdates = newTestCommitteeChain(tcBase, tfBase, true, 5, 10, 400, true) + tcFork = newTestCommitteeChain(tcBase, tfAlternative, true, 7, 10, 400, false) + tcAnotherGenesis = newTestCommitteeChain(nil, tfAnotherGenesis, true, 0, 10, 400, false) +) + +func TestCommitteeChainFixedCommitteeRoots(t *testing.T) { + for _, reload := range []bool{false, true} { + c := newCommitteeChainTest(t, tfBase, 300, true) + c.setClockPeriod(7) + c.addFixedCommitteeRoot(tcBase, 4, nil) + c.addFixedCommitteeRoot(tcBase, 5, nil) + c.addFixedCommitteeRoot(tcBase, 6, nil) + c.addFixedCommitteeRoot(tcBase, 8, ErrInvalidPeriod) // range has to be continuous + c.addFixedCommitteeRoot(tcBase, 3, nil) + c.addFixedCommitteeRoot(tcBase, 2, nil) + if reload { + c.reloadChain() + } + c.addCommittee(tcBase, 4, nil) + c.addCommittee(tcBase, 6, ErrInvalidPeriod) // range has to be continuous + c.addCommittee(tcBase, 5, nil) + c.addCommittee(tcBase, 6, nil) + c.addCommittee(tcAnotherGenesis, 3, ErrWrongCommitteeRoot) + c.addCommittee(tcBase, 3, nil) + if reload { + c.reloadChain() + } + c.verifyRange(tcBase, 3, 6) + } +} + +func TestCommitteeChainCheckpointSync(t *testing.T) { + for _, enforceTime := range []bool{false, true} { + for _, reload := range []bool{false, true} { + c := newCommitteeChainTest(t, tfBase, 300, enforceTime) + if enforceTime { + c.setClockPeriod(6) + } + c.insertUpdate(tcBase, 3, true, ErrInvalidPeriod) + c.addFixedCommitteeRoot(tcBase, 3, nil) + c.addFixedCommitteeRoot(tcBase, 4, nil) + c.insertUpdate(tcBase, 4, true, ErrInvalidPeriod) // still no committee + c.addCommittee(tcBase, 3, nil) + c.addCommittee(tcBase, 4, nil) + if reload { + c.reloadChain() + } + c.verifyRange(tcBase, 3, 4) + c.insertUpdate(tcBase, 3, false, nil) // update can be added without committee here + c.insertUpdate(tcBase, 4, false, ErrNeedCommittee) // but not here as committee 5 is not there yet + c.insertUpdate(tcBase, 4, true, nil) + c.verifyRange(tcBase, 3, 5) + c.insertUpdate(tcBaseWithInvalidUpdates, 5, true, ErrInvalidUpdate) // signer count too low + c.insertUpdate(tcBase, 5, true, nil) + if reload { + c.reloadChain() + } + if enforceTime { + c.insertUpdate(tcBase, 6, true, ErrInvalidUpdate) // future update rejected + c.setClockPeriod(7) + } + c.insertUpdate(tcBase, 6, true, nil) // when the time comes it's accepted + if reload { + c.reloadChain() + } + if enforceTime { + c.verifyRange(tcBase, 3, 6) // committee 7 is there but still in the future + c.setClockPeriod(8) + } + c.verifyRange(tcBase, 3, 7) // now period 7 can also be verified + // try reverse syncing an update + c.insertUpdate(tcBase, 2, false, ErrInvalidPeriod) // fixed committee is needed first + c.addFixedCommitteeRoot(tcBase, 2, nil) + c.addCommittee(tcBase, 2, nil) + c.insertUpdate(tcBase, 2, false, nil) + c.verifyRange(tcBase, 2, 7) + } + } +} + +func TestCommitteeChainReorg(t *testing.T) { + for _, reload := range []bool{false, true} { + for _, addBetterUpdates := range []bool{false, true} { + c := newCommitteeChainTest(t, tfBase, 300, true) + c.setClockPeriod(11) + c.addFixedCommitteeRoot(tcBase, 3, nil) + c.addFixedCommitteeRoot(tcBase, 4, nil) + c.addCommittee(tcBase, 3, nil) + for period := uint64(3); period < 10; period++ { + c.insertUpdate(tcBase, period, true, nil) + } + if reload { + c.reloadChain() + } + c.verifyRange(tcBase, 3, 10) + c.insertUpdate(tcReorgWithWorseUpdates, 5, true, ErrCannotReorg) + c.insertUpdate(tcReorgWithWorseUpdates2, 5, true, ErrCannotReorg) + if addBetterUpdates { + // add better updates for the base chain and expect first reorg to fail + // (only add updates as committees should be the same) + for period := uint64(5); period < 10; period++ { + c.insertUpdate(tcBaseWithBetterUpdates, period, false, nil) + } + if reload { + c.reloadChain() + } + c.verifyRange(tcBase, 3, 10) // still on the same chain + c.insertUpdate(tcReorgWithBetterUpdates, 5, true, ErrCannotReorg) + } else { + // reorg with better updates + c.insertUpdate(tcReorgWithBetterUpdates, 5, false, ErrNeedCommittee) + c.verifyRange(tcBase, 3, 10) // no success yet, still on the base chain + c.verifyRange(tcReorgWithBetterUpdates, 3, 5) + c.insertUpdate(tcReorgWithBetterUpdates, 5, true, nil) + // successful reorg, base chain should only match before the reorg period + if reload { + c.reloadChain() + } + c.verifyRange(tcBase, 3, 5) + c.verifyRange(tcReorgWithBetterUpdates, 3, 6) + for period := uint64(6); period < 10; period++ { + c.insertUpdate(tcReorgWithBetterUpdates, period, true, nil) + } + c.verifyRange(tcReorgWithBetterUpdates, 3, 10) + } + // reorg with finalized updates; should succeed even if base chain updates + // have been improved because a finalized update beats everything else + c.insertUpdate(tcReorgWithFinalizedUpdates, 5, false, ErrNeedCommittee) + c.insertUpdate(tcReorgWithFinalizedUpdates, 5, true, nil) + if reload { + c.reloadChain() + } + c.verifyRange(tcReorgWithFinalizedUpdates, 3, 6) + for period := uint64(6); period < 10; period++ { + c.insertUpdate(tcReorgWithFinalizedUpdates, period, true, nil) + } + c.verifyRange(tcReorgWithFinalizedUpdates, 3, 10) + } + } +} + +func TestCommitteeChainFork(t *testing.T) { + c := newCommitteeChainTest(t, tfAlternative, 300, true) + c.setClockPeriod(11) + // trying to sync a chain on an alternative fork with the base chain data + c.addFixedCommitteeRoot(tcBase, 0, nil) + c.addFixedCommitteeRoot(tcBase, 1, nil) + c.addCommittee(tcBase, 0, nil) + // shared section should sync without errors + for period := uint64(0); period < 7; period++ { + c.insertUpdate(tcBase, period, true, nil) + } + c.insertUpdate(tcBase, 7, true, ErrInvalidUpdate) // wrong fork + // committee root #7 is still the same but signatures are already signed with + // a different fork id so period 7 should only verify on the alternative fork + c.verifyRange(tcBase, 0, 6) + c.verifyRange(tcFork, 0, 7) + for period := uint64(7); period < 10; period++ { + c.insertUpdate(tcFork, period, true, nil) + } + c.verifyRange(tcFork, 0, 10) + // reload the chain while switching to the base fork + c.config = tfBase + c.reloadChain() + // updates 7..9 should be rolled back now + c.verifyRange(tcFork, 0, 6) // again, period 7 only verifies on the right fork + c.verifyRange(tcBase, 0, 7) + c.insertUpdate(tcFork, 7, true, ErrInvalidUpdate) // wrong fork + for period := uint64(7); period < 10; period++ { + c.insertUpdate(tcBase, period, true, nil) + } + c.verifyRange(tcBase, 0, 10) +} + +type committeeChainTest struct { + t *testing.T + db *memorydb.Database + clock *mclock.Simulated + config types.ChainConfig + signerThreshold int + enforceTime bool + chain *CommitteeChain +} + +func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThreshold int, enforceTime bool) *committeeChainTest { + c := &committeeChainTest{ + t: t, + db: memorydb.New(), + clock: &mclock.Simulated{}, + config: config, + signerThreshold: signerThreshold, + enforceTime: enforceTime, + } + c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) + return c +} + +func (c *committeeChainTest) reloadChain() { + c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) }) +} + +func (c *committeeChainTest) setClockPeriod(period float64) { + target := mclock.AbsTime(period * float64(time.Second*12*params.SyncPeriodLength)) + wait := time.Duration(target - c.clock.Now()) + if wait < 0 { + c.t.Fatalf("Invalid setClockPeriod") + } + c.clock.Run(wait) +} + +func (c *committeeChainTest) addFixedCommitteeRoot(tc *testCommitteeChain, period uint64, expErr error) { + if err := c.chain.addFixedCommitteeRoot(period, tc.periods[period].committee.Root()); err != expErr { + c.t.Errorf("Incorrect error output from addFixedCommitteeRoot at period %d (expected %v, got %v)", period, expErr, err) + } +} + +func (c *committeeChainTest) addCommittee(tc *testCommitteeChain, period uint64, expErr error) { + if err := c.chain.addCommittee(period, tc.periods[period].committee); err != expErr { + c.t.Errorf("Incorrect error output from addCommittee at period %d (expected %v, got %v)", period, expErr, err) + } +} + +func (c *committeeChainTest) insertUpdate(tc *testCommitteeChain, period uint64, addCommittee bool, expErr error) { + var committee *types.SerializedSyncCommittee + if addCommittee { + committee = tc.periods[period+1].committee + } + if err := c.chain.InsertUpdate(tc.periods[period].update, committee); err != expErr { + c.t.Errorf("Incorrect error output from InsertUpdate at period %d (expected %v, got %v)", period, expErr, err) + } +} + +func (c *committeeChainTest) verifySignedHeader(tc *testCommitteeChain, period float64, expOk bool) { + slot := uint64(period * float64(params.SyncPeriodLength)) + signedHead := GenerateTestSignedHeader(types.Header{Slot: slot}, &tc.config, tc.periods[types.SyncPeriod(slot)].committee, slot+1, 400) + if ok, _, _ := c.chain.VerifySignedHeader(signedHead); ok != expOk { + c.t.Errorf("Incorrect output from VerifySignedHeader at period %f (expected %v, got %v)", period, expOk, ok) + } +} + +func (c *committeeChainTest) verifyRange(tc *testCommitteeChain, begin, end uint64) { + if begin > 0 { + c.verifySignedHeader(tc, float64(begin)-0.5, false) + } + for period := begin; period <= end; period++ { + c.verifySignedHeader(tc, float64(period)+0.5, true) + } + c.verifySignedHeader(tc, float64(end)+1.5, false) +} + +func newTestGenesis() types.ChainConfig { + var config types.ChainConfig + rand.Read(config.GenesisValidatorsRoot[:]) + return config +} + +func newTestForks(config types.ChainConfig, forks types.Forks) types.ChainConfig { + for _, fork := range forks { + config.AddFork(fork.Name, fork.Epoch, fork.Version) + } + return config +} + +func newTestCommitteeChain(parent *testCommitteeChain, config types.ChainConfig, newCommittees bool, begin, end int, signerCount int, finalizedHeader bool) *testCommitteeChain { + tc := &testCommitteeChain{ + config: config, + } + if parent != nil { + tc.periods = make([]testPeriod, len(parent.periods)) + copy(tc.periods, parent.periods) + } + if newCommittees { + if begin == 0 { + tc.fillCommittees(begin, end+1) + } else { + tc.fillCommittees(begin+1, end+1) + } + } + tc.fillUpdates(begin, end, signerCount, finalizedHeader) + return tc +} + +type testPeriod struct { + committee *types.SerializedSyncCommittee + update *types.LightClientUpdate +} + +type testCommitteeChain struct { + periods []testPeriod + config types.ChainConfig +} + +func (tc *testCommitteeChain) fillCommittees(begin, end int) { + if len(tc.periods) <= end { + tc.periods = append(tc.periods, make([]testPeriod, end+1-len(tc.periods))...) + } + for i := begin; i <= end; i++ { + tc.periods[i].committee = GenerateTestCommittee() + } +} + +func (tc *testCommitteeChain) fillUpdates(begin, end int, signerCount int, finalizedHeader bool) { + for i := begin; i <= end; i++ { + tc.periods[i].update = GenerateTestUpdate(&tc.config, uint64(i), tc.periods[i].committee, tc.periods[i+1].committee, signerCount, finalizedHeader) + } +} diff --git a/beacon/light/range.go b/beacon/light/range.go new file mode 100644 index 0000000000..76ebe2381a --- /dev/null +++ b/beacon/light/range.go @@ -0,0 +1,78 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +// periodRange represents a (possibly zero-length) range of integers (sync periods). +type periodRange struct { + Start, End uint64 +} + +// isEmpty returns true if the length of the range is zero. +func (a periodRange) isEmpty() bool { + return a.End == a.Start +} + +// contains returns true if the range includes the given period. +func (a periodRange) contains(period uint64) bool { + return period >= a.Start && period < a.End +} + +// canExpand returns true if the range includes or can be expanded with the given +// period (either the range is empty or the given period is inside, right before or +// right after the range). +func (a periodRange) canExpand(period uint64) bool { + return a.isEmpty() || (period+1 >= a.Start && period <= a.End) +} + +// expand expands the range with the given period. +// This method assumes that canExpand returned true: otherwise this is a no-op. +func (a *periodRange) expand(period uint64) { + if a.isEmpty() { + a.Start, a.End = period, period+1 + return + } + if a.Start == period+1 { + a.Start-- + } + if a.End == period { + a.End++ + } +} + +// split splits the range into two ranges. The 'fromPeriod' will be the first +// element in the second range (if present). +// The original range is unchanged by this operation +func (a *periodRange) split(fromPeriod uint64) (periodRange, periodRange) { + if fromPeriod <= a.Start { + // First range empty, everything in second range, + return periodRange{}, *a + } + if fromPeriod >= a.End { + // Second range empty, everything in first range, + return *a, periodRange{} + } + x := periodRange{a.Start, fromPeriod} + y := periodRange{fromPeriod, a.End} + return x, y +} + +// each invokes the supplied function fn once per period in range +func (a *periodRange) each(fn func(uint64)) { + for p := a.Start; p < a.End; p++ { + fn(p) + } +} diff --git a/beacon/light/test_helpers.go b/beacon/light/test_helpers.go new file mode 100644 index 0000000000..f537d963a6 --- /dev/null +++ b/beacon/light/test_helpers.go @@ -0,0 +1,152 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package light + +import ( + "crypto/rand" + "crypto/sha256" + mrand "math/rand" + + "github.com/ethereum/go-ethereum/beacon/merkle" + "github.com/ethereum/go-ethereum/beacon/params" + "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" +) + +func GenerateTestCommittee() *types.SerializedSyncCommittee { + s := new(types.SerializedSyncCommittee) + rand.Read(s[:32]) + return s +} + +func GenerateTestUpdate(config *types.ChainConfig, period uint64, committee, nextCommittee *types.SerializedSyncCommittee, signerCount int, finalizedHeader bool) *types.LightClientUpdate { + update := new(types.LightClientUpdate) + update.NextSyncCommitteeRoot = nextCommittee.Root() + var attestedHeader types.Header + if finalizedHeader { + update.FinalizedHeader = new(types.Header) + *update.FinalizedHeader, update.NextSyncCommitteeBranch = makeTestHeaderWithMerkleProof(types.SyncPeriodStart(period)+100, params.StateIndexNextSyncCommittee, merkle.Value(update.NextSyncCommitteeRoot)) + attestedHeader, update.FinalityBranch = makeTestHeaderWithMerkleProof(types.SyncPeriodStart(period)+200, params.StateIndexFinalBlock, merkle.Value(update.FinalizedHeader.Hash())) + } else { + attestedHeader, update.NextSyncCommitteeBranch = makeTestHeaderWithMerkleProof(types.SyncPeriodStart(period)+2000, params.StateIndexNextSyncCommittee, merkle.Value(update.NextSyncCommitteeRoot)) + } + update.AttestedHeader = GenerateTestSignedHeader(attestedHeader, config, committee, attestedHeader.Slot+1, signerCount) + return update +} + +func GenerateTestSignedHeader(header types.Header, config *types.ChainConfig, committee *types.SerializedSyncCommittee, signatureSlot uint64, signerCount int) types.SignedHeader { + bitmask := makeBitmask(signerCount) + signingRoot, _ := config.Forks.SigningRoot(header) + c, _ := dummyVerifier{}.deserializeSyncCommittee(committee) + return types.SignedHeader{ + Header: header, + Signature: types.SyncAggregate{ + Signers: bitmask, + Signature: makeDummySignature(c.(dummySyncCommittee), signingRoot, bitmask), + }, + SignatureSlot: signatureSlot, + } +} + +func GenerateTestCheckpoint(period uint64, committee *types.SerializedSyncCommittee) *types.BootstrapData { + header, branch := makeTestHeaderWithMerkleProof(types.SyncPeriodStart(period)+200, params.StateIndexSyncCommittee, merkle.Value(committee.Root())) + return &types.BootstrapData{ + Header: header, + Committee: committee, + CommitteeRoot: committee.Root(), + CommitteeBranch: branch, + } +} + +func makeBitmask(signerCount int) (bitmask [params.SyncCommitteeBitmaskSize]byte) { + for i := 0; i < params.SyncCommitteeSize; i++ { + if mrand.Intn(params.SyncCommitteeSize-i) < signerCount { + bitmask[i/8] += byte(1) << (i & 7) + signerCount-- + } + } + return +} + +func makeTestHeaderWithMerkleProof(slot, index uint64, value merkle.Value) (types.Header, merkle.Values) { + var branch merkle.Values + hasher := sha256.New() + for index > 1 { + var proofHash merkle.Value + rand.Read(proofHash[:]) + hasher.Reset() + if index&1 == 0 { + hasher.Write(value[:]) + hasher.Write(proofHash[:]) + } else { + hasher.Write(proofHash[:]) + hasher.Write(value[:]) + } + hasher.Sum(value[:0]) + index >>= 1 + branch = append(branch, proofHash) + } + return types.Header{Slot: slot, StateRoot: common.Hash(value)}, branch +} + +// syncCommittee holds either a blsSyncCommittee or a fake dummySyncCommittee used for testing +type syncCommittee interface{} + +// committeeSigVerifier verifies sync committee signatures (either proper BLS +// signatures or fake signatures used for testing) +type committeeSigVerifier interface { + deserializeSyncCommittee(s *types.SerializedSyncCommittee) (syncCommittee, error) + verifySignature(committee syncCommittee, signedRoot common.Hash, aggregate *types.SyncAggregate) bool +} + +// blsVerifier implements committeeSigVerifier +type blsVerifier struct{} + +// deserializeSyncCommittee implements committeeSigVerifier +func (blsVerifier) deserializeSyncCommittee(s *types.SerializedSyncCommittee) (syncCommittee, error) { + return s.Deserialize() +} + +// verifySignature implements committeeSigVerifier +func (blsVerifier) verifySignature(committee syncCommittee, signingRoot common.Hash, aggregate *types.SyncAggregate) bool { + return committee.(*types.SyncCommittee).VerifySignature(signingRoot, aggregate) +} + +type dummySyncCommittee [32]byte + +// dummyVerifier implements committeeSigVerifier +type dummyVerifier struct{} + +// deserializeSyncCommittee implements committeeSigVerifier +func (dummyVerifier) deserializeSyncCommittee(s *types.SerializedSyncCommittee) (syncCommittee, error) { + var sc dummySyncCommittee + copy(sc[:], s[:32]) + return sc, nil +} + +// verifySignature implements committeeSigVerifier +func (dummyVerifier) verifySignature(committee syncCommittee, signingRoot common.Hash, aggregate *types.SyncAggregate) bool { + return aggregate.Signature == makeDummySignature(committee.(dummySyncCommittee), signingRoot, aggregate.Signers) +} + +func makeDummySignature(committee dummySyncCommittee, signingRoot common.Hash, bitmask [params.SyncCommitteeBitmaskSize]byte) (sig [params.BLSSignatureSize]byte) { + for i, b := range committee[:] { + sig[i] = b ^ signingRoot[i] + } + copy(sig[32:], bitmask[:]) + return +} diff --git a/beacon/types/update.go b/beacon/types/light_sync.go similarity index 88% rename from beacon/types/update.go rename to beacon/types/light_sync.go index 06c1b61792..3284081e4d 100644 --- a/beacon/types/update.go +++ b/beacon/types/light_sync.go @@ -25,6 +25,24 @@ import ( "github.com/ethereum/go-ethereum/common" ) +// BootstrapData contains a sync committee where light sync can be started, +// together with a proof through a beacon header and corresponding state. +// Note: BootstrapData is fetched from a server based on a known checkpoint hash. +type BootstrapData struct { + Header Header + CommitteeRoot common.Hash + Committee *SerializedSyncCommittee `rlp:"-"` + CommitteeBranch merkle.Values +} + +// Validate verifies the proof included in BootstrapData. +func (c *BootstrapData) Validate() error { + if c.CommitteeRoot != c.Committee.Root() { + return errors.New("wrong committee root") + } + return merkle.VerifyProof(c.Header.StateRoot, params.StateIndexSyncCommittee, c.CommitteeBranch, merkle.Value(c.CommitteeRoot)) +} + // LightClientUpdate is a proof of the next sync committee root based on a header // signed by the sync committee of the given period. Optionally, the update can // prove quasi-finality by the signed header referring to a previous, finalized diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 8e82459e82..be03723553 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -132,6 +132,10 @@ var ( CliqueSnapshotPrefix = []byte("clique-") + BestUpdateKey = []byte("update-") // bigEndian64(syncPeriod) -> RLP(types.LightClientUpdate) (nextCommittee only referenced by root hash) + FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash + SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) From d98d70f670297a4bfa86db1a67a9c024f7186f43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 8 Dec 2023 15:16:04 +0200 Subject: [PATCH 14/15] cmd/utils, eth: disallow invalid snap sync / snapshot flag combos (#28657) * eth: prevent startup in snap mode without snapshots * cmd/utils: try to fix bad flag combos wrt snap sync and snapshot generation --- cmd/utils/flags.go | 10 ++++++++-- eth/handler.go | 4 ++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 27e1b3f623..d4c918bf4f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1677,10 +1677,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheLogSizeFlag.Name) { cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) } - if !ctx.Bool(SnapshotFlag.Name) { + if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { - log.Info("Snap sync requested, enabling --snapshot") + if !ctx.Bool(SnapshotFlag.Name) { + log.Warn("Snap sync requested, enabling --snapshot") + } + if cfg.SnapshotCache == 0 { + log.Warn("Snap sync requested, resetting --cache.snapshot") + cfg.SnapshotCache = ctx.Int(CacheFlag.Name) * CacheSnapshotFlag.Value / 100 + } } else { cfg.TrieCleanCache += cfg.SnapshotCache cfg.SnapshotCache = 0 // Disabled diff --git a/eth/handler.go b/eth/handler.go index f0021e5644..a327af6113 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -178,6 +178,10 @@ func newHandler(config *handlerConfig) (*handler, error) { log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash()) } } + // If snap sync is requested but snapshots are disabled, fail loudly + if h.snapSync.Load() && config.Chain.Snapshots() == nil { + return nil, errors.New("snap sync not supported with snapshots disabled") + } // Construct the downloader (long sync) h.downloader = downloader.New(config.Database, h.eventMux, h.chain, nil, h.removePeer, h.enableSyncedFeatures) if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { From e206d3f8975bd98cc86d14055dca40f996bacc60 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 8 Dec 2023 21:28:23 +0800 Subject: [PATCH 15/15] trie: remove inconsistent trie nodes during sync in path mode (#28595) This fixes a database corruption issue that could occur during state healing. When sync is aborted while certain modifications were already committed, and a reorg occurs, the database would contain incorrect trie nodes stored by path. These nodes need to detected/deleted in order to obtain a complete and fully correct state after state healing. --------- Co-authored-by: Felix Lange --- ethdb/dbtest/testsuite.go | 6 +- trie/sync.go | 223 ++++++++++++++++++++++++-------------- trie/sync_test.go | 125 ++++++++++++++++++++- 3 files changed, 270 insertions(+), 84 deletions(-) diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 0d3d5f5aa6..29bd24364e 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -273,9 +273,13 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { b.Put([]byte("5"), nil) b.Delete([]byte("1")) b.Put([]byte("6"), nil) - b.Delete([]byte("3")) + + b.Delete([]byte("3")) // delete then put b.Put([]byte("3"), nil) + b.Put([]byte("7"), nil) // put then delete + b.Delete([]byte("7")) + if err := b.Write(); err != nil { t.Fatal(err) } diff --git a/trie/sync.go b/trie/sync.go index 8eaed9f21a..589d28364b 100644 --- a/trie/sync.go +++ b/trie/sync.go @@ -116,10 +116,9 @@ type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Ha // nodeRequest represents a scheduled or already in-flight trie node retrieval request. type nodeRequest struct { - hash common.Hash // Hash of the trie node to retrieve - path []byte // Merkle path leading to this node for prioritization - data []byte // Data content of the node, cached until all subtrees complete - deletes [][]byte // List of internal path segments for trie nodes to delete + hash common.Hash // Hash of the trie node to retrieve + path []byte // Merkle path leading to this node for prioritization + data []byte // Data content of the node, cached until all subtrees complete parent *nodeRequest // Parent state node referencing this entry deps int // Number of dependencies before allowed to commit this node @@ -146,38 +145,85 @@ type CodeSyncResult struct { Data []byte // Data content of the retrieved bytecode } +// nodeOp represents an operation upon the trie node. It can either represent a +// deletion to the specific node or a node write for persisting retrieved node. +type nodeOp struct { + owner common.Hash // identifier of the trie (empty for account trie) + path []byte // path from the root to the specified node. + blob []byte // the content of the node (nil for deletion) + hash common.Hash // hash of the node content (empty for node deletion) +} + +// isDelete indicates if the operation is a database deletion. +func (op *nodeOp) isDelete() bool { + return len(op.blob) == 0 +} + // syncMemBatch is an in-memory buffer of successfully downloaded but not yet // persisted data items. type syncMemBatch struct { - nodes map[string][]byte // In-memory membatch of recently completed nodes - hashes map[string]common.Hash // Hashes of recently completed nodes - deletes map[string]struct{} // List of paths for trie node to delete - codes map[common.Hash][]byte // In-memory membatch of recently completed codes - size uint64 // Estimated batch-size of in-memory data. + scheme string // State scheme identifier + codes map[common.Hash][]byte // In-memory batch of recently completed codes + nodes []nodeOp // In-memory batch of recently completed/deleted nodes + size uint64 // Estimated batch-size of in-memory data. } // newSyncMemBatch allocates a new memory-buffer for not-yet persisted trie nodes. -func newSyncMemBatch() *syncMemBatch { +func newSyncMemBatch(scheme string) *syncMemBatch { return &syncMemBatch{ - nodes: make(map[string][]byte), - hashes: make(map[string]common.Hash), - deletes: make(map[string]struct{}), - codes: make(map[common.Hash][]byte), + scheme: scheme, + codes: make(map[common.Hash][]byte), } } -// hasNode reports the trie node with specific path is already cached. -func (batch *syncMemBatch) hasNode(path []byte) bool { - _, ok := batch.nodes[string(path)] - return ok -} - // hasCode reports the contract code with specific hash is already cached. func (batch *syncMemBatch) hasCode(hash common.Hash) bool { _, ok := batch.codes[hash] return ok } +// addCode caches a contract code database write operation. +func (batch *syncMemBatch) addCode(hash common.Hash, code []byte) { + batch.codes[hash] = code + batch.size += common.HashLength + uint64(len(code)) +} + +// addNode caches a node database write operation. +func (batch *syncMemBatch) addNode(owner common.Hash, path []byte, blob []byte, hash common.Hash) { + if batch.scheme == rawdb.PathScheme { + if owner == (common.Hash{}) { + batch.size += uint64(len(path) + len(blob)) + } else { + batch.size += common.HashLength + uint64(len(path)+len(blob)) + } + } else { + batch.size += common.HashLength + uint64(len(blob)) + } + batch.nodes = append(batch.nodes, nodeOp{ + owner: owner, + path: path, + blob: blob, + hash: hash, + }) +} + +// delNode caches a node database delete operation. +func (batch *syncMemBatch) delNode(owner common.Hash, path []byte) { + if batch.scheme != rawdb.PathScheme { + log.Error("Unexpected node deletion", "owner", owner, "path", path, "scheme", batch.scheme) + return // deletion is not supported in hash mode. + } + if owner == (common.Hash{}) { + batch.size += uint64(len(path)) + } else { + batch.size += common.HashLength + uint64(len(path)) + } + batch.nodes = append(batch.nodes, nodeOp{ + owner: owner, + path: path, + }) +} + // Sync is the main state trie synchronisation scheduler, which provides yet // unknown trie hashes to retrieve, accepts node data associated with said hashes // and reconstructs the trie step by step until all is done. @@ -196,7 +242,7 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb ts := &Sync{ scheme: scheme, database: database, - membatch: newSyncMemBatch(), + membatch: newSyncMemBatch(scheme), nodeReqs: make(map[string]*nodeRequest), codeReqs: make(map[common.Hash]*codeRequest), queue: prque.New[int64, any](nil), // Ugh, can contain both string and hash, whyyy @@ -210,16 +256,17 @@ func NewSync(root common.Hash, database ethdb.KeyValueReader, callback LeafCallb // parent for completion tracking. The given path is a unique node path in // hex format and contain all the parent path if it's layered trie node. func (s *Sync) AddSubTrie(root common.Hash, path []byte, parent common.Hash, parentPath []byte, callback LeafCallback) { - // Short circuit if the trie is empty or already known if root == types.EmptyRootHash { return } - if s.membatch.hasNode(path) { - return - } owner, inner := ResolvePath(path) - if rawdb.HasTrieNode(s.database, owner, inner, root, s.scheme) { + exist, inconsistent := s.hasNode(owner, inner, root) + if exist { + // The entire subtrie is already present in the database. return + } else if inconsistent { + // There is a pre-existing node with the wrong hash in DB, remove it. + s.membatch.delNode(owner, inner) } // Assemble the new sub-trie sync request req := &nodeRequest{ @@ -371,39 +418,42 @@ func (s *Sync) ProcessNode(result NodeSyncResult) error { } // Commit flushes the data stored in the internal membatch out to persistent -// storage, returning any occurred error. +// storage, returning any occurred error. The whole data set will be flushed +// in an atomic database batch. func (s *Sync) Commit(dbw ethdb.Batch) error { // Flush the pending node writes into database batch. var ( account int storage int ) - for path, value := range s.membatch.nodes { - owner, inner := ResolvePath([]byte(path)) - if owner == (common.Hash{}) { - account += 1 + for _, op := range s.membatch.nodes { + if op.isDelete() { + // node deletion is only supported in path mode. + if op.owner == (common.Hash{}) { + rawdb.DeleteAccountTrieNode(dbw, op.path) + } else { + rawdb.DeleteStorageTrieNode(dbw, op.owner, op.path) + } + deletionGauge.Inc(1) } else { - storage += 1 + if op.owner == (common.Hash{}) { + account += 1 + } else { + storage += 1 + } + rawdb.WriteTrieNode(dbw, op.owner, op.path, op.hash, op.blob, s.scheme) } - rawdb.WriteTrieNode(dbw, owner, inner, s.membatch.hashes[path], value, s.scheme) } accountNodeSyncedGauge.Inc(int64(account)) storageNodeSyncedGauge.Inc(int64(storage)) - // Flush the pending node deletes into the database batch. - // Please note that each written and deleted node has a - // unique path, ensuring no duplication occurs. - for path := range s.membatch.deletes { - owner, inner := ResolvePath([]byte(path)) - rawdb.DeleteTrieNode(dbw, owner, inner, common.Hash{} /* unused */, s.scheme) - } // Flush the pending code writes into database batch. for hash, value := range s.membatch.codes { rawdb.WriteCode(dbw, hash, value) } codeSyncedGauge.Inc(int64(len(s.membatch.codes))) - s.membatch = newSyncMemBatch() // reset the batch + s.membatch = newSyncMemBatch(s.scheme) // reset the batch return nil } @@ -476,12 +526,15 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // child as invalid. This is essential in the case of path mode // scheme; otherwise, state healing might overwrite existing child // nodes silently while leaving a dangling parent node within the - // range of this internal path on disk. This would break the - // guarantee for state healing. + // range of this internal path on disk and the persistent state + // ends up with a very weird situation that nodes on the same path + // are not inconsistent while they all present in disk. This property + // would break the guarantee for state healing. // // While it's possible for this shortNode to overwrite a previously // existing full node, the other branches of the fullNode can be - // retained as they remain untouched and complete. + // retained as they are not accessible with the new shortNode, and + // also the whole sub-trie is still untouched and complete. // // This step is only necessary for path mode, as there is no deletion // in hash mode at all. @@ -498,8 +551,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { exists = rawdb.ExistsStorageTrieNode(s.database, owner, append(inner, key[:i]...)) } if exists { - req.deletes = append(req.deletes, key[:i]) - deletionGauge.Inc(1) + s.membatch.delNode(owner, append(inner, key[:i]...)) log.Debug("Detected dangling node", "owner", owner, "path", append(inner, key[:i]...)) } } @@ -521,6 +573,7 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { var ( missing = make(chan *nodeRequest, len(children)) pending sync.WaitGroup + batchMu sync.Mutex ) for _, child := range children { // Notify any external watcher of a new key/value node @@ -538,34 +591,32 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { } } } - // If the child references another node, resolve or schedule + // If the child references another node, resolve or schedule. + // We check all children concurrently. if node, ok := (child.node).(hashNode); ok { - // Try to resolve the node from the local database - if s.membatch.hasNode(child.path) { - continue - } - // Check the presence of children concurrently + path := child.path + hash := common.BytesToHash(node) pending.Add(1) - go func(child childNode) { + go func() { defer pending.Done() - - // If database says duplicate, then at least the trie node is present - // and we hold the assumption that it's NOT legacy contract code. - var ( - chash = common.BytesToHash(node) - owner, inner = ResolvePath(child.path) - ) - if rawdb.HasTrieNode(s.database, owner, inner, chash, s.scheme) { + owner, inner := ResolvePath(path) + exist, inconsistent := s.hasNode(owner, inner, hash) + if exist { return + } else if inconsistent { + // There is a pre-existing node with the wrong hash in DB, remove it. + batchMu.Lock() + s.membatch.delNode(owner, inner) + batchMu.Unlock() } // Locally unknown node, schedule for retrieval missing <- &nodeRequest{ - path: child.path, - hash: chash, + path: path, + hash: hash, parent: req, callback: req.callback, } - }(child) + }() } } pending.Wait() @@ -587,21 +638,10 @@ func (s *Sync) children(req *nodeRequest, object node) ([]*nodeRequest, error) { // committed themselves. func (s *Sync) commitNodeRequest(req *nodeRequest) error { // Write the node content to the membatch - s.membatch.nodes[string(req.path)] = req.data - s.membatch.hashes[string(req.path)] = req.hash + owner, path := ResolvePath(req.path) + s.membatch.addNode(owner, path, req.data, req.hash) - // The size tracking refers to the db-batch, not the in-memory data. - if s.scheme == rawdb.PathScheme { - s.membatch.size += uint64(len(req.path) + len(req.data)) - } else { - s.membatch.size += common.HashLength + uint64(len(req.data)) - } - // Delete the internal nodes which are marked as invalid - for _, segment := range req.deletes { - path := append(req.path, segment...) - s.membatch.deletes[string(path)] = struct{}{} - s.membatch.size += uint64(len(path)) - } + // Removed the completed node request delete(s.nodeReqs, string(req.path)) s.fetches[len(req.path)]-- @@ -622,8 +662,9 @@ func (s *Sync) commitNodeRequest(req *nodeRequest) error { // committed themselves. func (s *Sync) commitCodeRequest(req *codeRequest) error { // Write the node content to the membatch - s.membatch.codes[req.hash] = req.data - s.membatch.size += common.HashLength + uint64(len(req.data)) + s.membatch.addCode(req.hash, req.data) + + // Removed the completed code request delete(s.codeReqs, req.hash) s.fetches[len(req.path)]-- @@ -639,6 +680,28 @@ func (s *Sync) commitCodeRequest(req *codeRequest) error { return nil } +// hasNode reports whether the specified trie node is present in the database. +// 'exists' is true when the node exists in the database and matches the given root +// hash. The 'inconsistent' return value is true when the node exists but does not +// match the expected hash. +func (s *Sync) hasNode(owner common.Hash, path []byte, hash common.Hash) (exists bool, inconsistent bool) { + // If node is running with hash scheme, check the presence with node hash. + if s.scheme == rawdb.HashScheme { + return rawdb.HasLegacyTrieNode(s.database, hash), false + } + // If node is running with path scheme, check the presence with node path. + var blob []byte + var dbHash common.Hash + if owner == (common.Hash{}) { + blob, dbHash = rawdb.ReadAccountTrieNode(s.database, path) + } else { + blob, dbHash = rawdb.ReadStorageTrieNode(s.database, owner, path) + } + exists = hash == dbHash + inconsistent = !exists && len(blob) != 0 + return exists, inconsistent +} + // ResolvePath resolves the provided composite node path by separating the // path in account trie if it's existent. func ResolvePath(path []byte) (common.Hash, []byte) { diff --git a/trie/sync_test.go b/trie/sync_test.go index 5edfb32a37..585181b48c 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -684,8 +684,11 @@ func testSyncOrdering(t *testing.T, scheme string) { } } } - func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database) { + syncWithHookWriter(t, root, db, srcDb, nil) +} + +func syncWithHookWriter(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database, hookWriter ethdb.KeyValueWriter) { // Create a destination trie and sync with the scheduler sched := NewSync(root, db, nil, srcDb.Scheme()) @@ -723,8 +726,11 @@ func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database if err := sched.Commit(batch); err != nil { t.Fatalf("failed to commit data: %v", err) } - batch.Write() - + if hookWriter != nil { + batch.Replay(hookWriter) + } else { + batch.Write() + } paths, nodes, _ = sched.Missing(0) elements = elements[:0] for i := 0; i < len(paths); i++ { @@ -894,3 +900,116 @@ func testPivotMove(t *testing.T, scheme string, tiny bool) { syncWith(t, rootC, destDisk, srcTrieDB) checkTrieContents(t, destDisk, scheme, srcTrie.Hash().Bytes(), stateC, true) } + +func TestSyncAbort(t *testing.T) { + testSyncAbort(t, rawdb.PathScheme) + testSyncAbort(t, rawdb.HashScheme) +} + +type hookWriter struct { + db ethdb.KeyValueStore + filter func(key []byte, value []byte) bool +} + +// Put inserts the given value into the key-value data store. +func (w *hookWriter) Put(key []byte, value []byte) error { + if w.filter != nil && w.filter(key, value) { + return nil + } + return w.db.Put(key, value) +} + +// Delete removes the key from the key-value data store. +func (w *hookWriter) Delete(key []byte) error { + return w.db.Delete(key) +} + +func testSyncAbort(t *testing.T, scheme string) { + var ( + srcDisk = rawdb.NewMemoryDatabase() + srcTrieDB = newTestDatabase(srcDisk, scheme) + srcTrie, _ = New(TrieID(types.EmptyRootHash), srcTrieDB) + + deleteFn = func(key []byte, tr *Trie, states map[string][]byte) { + tr.Delete(key) + delete(states, string(key)) + } + writeFn = func(key []byte, val []byte, tr *Trie, states map[string][]byte) { + if val == nil { + val = randBytes(32) + } + tr.Update(key, val) + states[string(key)] = common.CopyBytes(val) + } + copyStates = func(states map[string][]byte) map[string][]byte { + cpy := make(map[string][]byte) + for k, v := range states { + cpy[k] = v + } + return cpy + } + ) + var ( + stateA = make(map[string][]byte) + key = randBytes(32) + val = randBytes(32) + ) + for i := 0; i < 256; i++ { + writeFn(randBytes(32), nil, srcTrie, stateA) + } + writeFn(key, val, srcTrie, stateA) + + rootA, nodesA, _ := srcTrie.Commit(false) + if err := srcTrieDB.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil); err != nil { + panic(err) + } + if err := srcTrieDB.Commit(rootA, false); err != nil { + panic(err) + } + // Create a destination trie and sync with the scheduler + destDisk := rawdb.NewMemoryDatabase() + syncWith(t, rootA, destDisk, srcTrieDB) + checkTrieContents(t, destDisk, scheme, srcTrie.Hash().Bytes(), stateA, true) + + // Delete the element from the trie + stateB := copyStates(stateA) + srcTrie, _ = New(TrieID(rootA), srcTrieDB) + deleteFn(key, srcTrie, stateB) + + rootB, nodesB, _ := srcTrie.Commit(false) + if err := srcTrieDB.Update(rootB, rootA, 0, trienode.NewWithNodeSet(nodesB), nil); err != nil { + panic(err) + } + if err := srcTrieDB.Commit(rootB, false); err != nil { + panic(err) + } + + // Sync the new state, but never persist the new root node. Before the + // fix #28595, the original old root node will still be left in database + // which breaks the next healing cycle. + syncWithHookWriter(t, rootB, destDisk, srcTrieDB, &hookWriter{db: destDisk, filter: func(key []byte, value []byte) bool { + if scheme == rawdb.HashScheme { + return false + } + if len(value) == 0 { + return false + } + ok, path := rawdb.ResolveAccountTrieNodeKey(key) + return ok && len(path) == 0 + }}) + + // Add elements to expand trie + stateC := copyStates(stateB) + srcTrie, _ = New(TrieID(rootB), srcTrieDB) + + writeFn(key, val, srcTrie, stateC) + rootC, nodesC, _ := srcTrie.Commit(false) + if err := srcTrieDB.Update(rootC, rootB, 0, trienode.NewWithNodeSet(nodesC), nil); err != nil { + panic(err) + } + if err := srcTrieDB.Commit(rootC, false); err != nil { + panic(err) + } + syncWith(t, rootC, destDisk, srcTrieDB) + checkTrieContents(t, destDisk, scheme, srcTrie.Hash().Bytes(), stateC, true) +}