From 2a2013014c46844728421a1acc5ad40ca823414b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 24 Nov 2023 13:26:42 +0200 Subject: [PATCH 01/43] eth, internal/ethapi: drop some weird indirection (#28597) --- eth/api_backend.go | 4 ++-- internal/ethapi/api.go | 6 +++--- internal/ethapi/api_test.go | 5 ++--- internal/ethapi/backend.go | 2 +- internal/ethapi/transaction_args_test.go | 4 ++-- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index 601e555158..84eb200095 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -249,7 +249,7 @@ func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) (*vm.EVM, func() error) { +func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.eth.blockchain.GetVMConfig() } @@ -260,7 +260,7 @@ func (b *EthAPIBackend) GetEVM(ctx context.Context, msg *core.Message, state *st } else { context = core.NewEVMBlockContext(header, b.eth.BlockChain(), nil) } - return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig), state.Error + return vm.NewEVM(context, txContext, state, b.eth.blockchain.Config(), *vmConfig) } func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 38a7924124..89c132ddee 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1083,7 +1083,7 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S if blockOverrides != nil { blockOverrides.Apply(&blockCtx) } - evm, vmError := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) + evm := b.GetEVM(ctx, msg, state, header, &vm.Config{NoBaseFee: true}, &blockCtx) // Wait for the context to be done and cancel the evm. Even if the // EVM has finished, cancelling may be done (repeatedly) @@ -1095,7 +1095,7 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S // Execute the message. gp := new(core.GasPool).AddGas(math.MaxUint64) result, err := core.ApplyMessage(evm, msg, gp) - if err := vmError(); err != nil { + if err := state.Error(); err != nil { return nil, err } @@ -1640,7 +1640,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Apply the transaction with the access list tracer tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) config := vm.Config{Tracer: tracer, NoBaseFee: true} - vmenv, _ := b.GetEVM(ctx, msg, statedb, header, &config, nil) + vmenv := b.GetEVM(ctx, msg, statedb, header, &config, nil) res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.toTransaction().Hash(), err) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index a67bd1203b..4a7694737e 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -536,8 +536,7 @@ func (b testBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { } return big.NewInt(1) } -func (b testBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) (*vm.EVM, func() error) { - vmError := func() error { return nil } +func (b testBackend) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockContext *vm.BlockContext) *vm.EVM { if vmConfig == nil { vmConfig = b.chain.GetVMConfig() } @@ -546,7 +545,7 @@ func (b testBackend) GetEVM(ctx context.Context, msg *core.Message, state *state if blockContext != nil { context = *blockContext } - return vm.NewEVM(context, txContext, state, b.chain.Config(), *vmConfig), vmError + return vm.NewEVM(context, txContext, state, b.chain.Config(), *vmConfig) } func (b testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { panic("implement me") diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 458fb811ed..50f338f5ca 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -68,7 +68,7 @@ type Backend interface { PendingBlockAndReceipts() (*types.Block, types.Receipts) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) GetTd(ctx context.Context, hash common.Hash) *big.Int - GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) (*vm.EVM, func() error) + GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 9161d5e681..9dc58bdeb5 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -305,8 +305,8 @@ func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number return nil, nil } func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } -func (b *backendMock) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) (*vm.EVM, func() error) { - return nil, nil +func (b *backendMock) GetEVM(ctx context.Context, msg *core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config, blockCtx *vm.BlockContext) *vm.EVM { + return nil } func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } func (b *backendMock) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { From 333dd956bfdf1d5086d38cceedbba25a366fb6ac Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Sat, 25 Nov 2023 21:56:22 +0800 Subject: [PATCH 02/43] trie: fix random test generator early terminate (#28590) This change fixes a minor bug in the `randTest.Generate` function, which caused the `quick.Check` to be a no-op. --- trie/trie_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/trie/trie_test.go b/trie/trie_test.go index 6af0f67b9f..c5bd3faf53 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -372,6 +372,9 @@ func TestRandomCases(t *testing.T) { // Instances of this test are created by Generate. type randTest []randTestStep +// compile-time interface check +var _ quick.Generator = (randTest)(nil) + type randTestStep struct { op int key []byte // for opUpdate, opDelete, opGet @@ -394,7 +397,7 @@ const ( func (randTest) Generate(r *rand.Rand, size int) reflect.Value { var finishedFn = func() bool { size-- - return size > 0 + return size == 0 } return reflect.ValueOf(generateSteps(finishedFn, r)) } From 1e28e0bb03c6911a691d7125cb8c5e41faac2eab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 27 Nov 2023 16:20:09 +0200 Subject: [PATCH 03/43] eth/gasestimator, internal/ethapi: move gas estimator out of rpc (#28600) --- eth/gasestimator/gasestimator.go | 194 +++++++++++++++++++++++++++++++ internal/ethapi/api.go | 149 ++++-------------------- internal/ethapi/api_test.go | 8 +- 3 files changed, 222 insertions(+), 129 deletions(-) create mode 100644 eth/gasestimator/gasestimator.go diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go new file mode 100644 index 0000000000..3e74b5b08b --- /dev/null +++ b/eth/gasestimator/gasestimator.go @@ -0,0 +1,194 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package gasestimator + +import ( + "context" + "errors" + "fmt" + "math" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" +) + +// Options are the contextual parameters to execute the requested call. +// +// Whilst it would be possible to pass a blockchain object that aggregates all +// these together, it would be excessively hard to test. Splitting the parts out +// allows testing without needing a proper live chain. +type Options struct { + Config *params.ChainConfig // Chain configuration for hard fork selection + Chain core.ChainContext // Chain context to access past block hashes + Header *types.Header // Header defining the block context to execute in + State *state.StateDB // Pre-state on top of which to estimate the gas +} + +// Estimate returns the lowest possible gas limit that allows the transaction to +// run successfully with the provided context optons. It returns an error if the +// transaction would always revert, or if there are unexpected failures. +func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uint64) (uint64, []byte, error) { + // Binary search the gas limit, as it may need to be higher than the amount used + var ( + lo uint64 // lowest-known gas limit where tx execution fails + hi uint64 // lowest-known gas limit where tx execution succeeds + ) + // Determine the highest gas limit can be used during the estimation. + hi = opts.Header.GasLimit + if call.GasLimit >= params.TxGas { + hi = call.GasLimit + } + // Normalize the max fee per gas the call is willing to spend. + var feeCap *big.Int + if call.GasFeeCap != nil { + feeCap = call.GasFeeCap + } else if call.GasPrice != nil { + feeCap = call.GasPrice + } else { + feeCap = common.Big0 + } + // Recap the highest gas limit with account's available balance. + if feeCap.BitLen() != 0 { + balance := opts.State.GetBalance(call.From) + + available := new(big.Int).Set(balance) + if call.Value != nil { + if call.Value.Cmp(available) >= 0 { + return 0, nil, core.ErrInsufficientFundsForTransfer + } + available.Sub(available, call.Value) + } + allowance := new(big.Int).Div(available, feeCap) + + // If the allowance is larger than maximum uint64, skip checking + if allowance.IsUint64() && hi > allowance.Uint64() { + transfer := call.Value + if transfer == nil { + transfer = new(big.Int) + } + log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, + "sent", transfer, "maxFeePerGas", feeCap, "fundable", allowance) + hi = allowance.Uint64() + } + } + // Recap the highest gas allowance with specified gascap. + if gasCap != 0 && hi > gasCap { + log.Warn("Caller gas above allowance, capping", "requested", hi, "cap", gasCap) + hi = gasCap + } + // We first execute the transaction at the highest allowable gas limit, since if this fails we + // can return error immediately. + failed, result, err := execute(ctx, call, opts, hi) + if err != nil { + return 0, nil, err + } + if failed { + if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) { + return 0, result.Revert(), result.Err + } + return 0, nil, fmt.Errorf("gas required exceeds allowance (%d)", hi) + } + // For almost any transaction, the gas consumed by the unconstrained execution + // above lower-bounds the gas limit required for it to succeed. One exception + // is those that explicitly check gas remaining in order to execute within a + // given limit, but we probably don't want to return the lowest possible gas + // limit for these cases anyway. + lo = result.UsedGas - 1 + + // Binary search for the smallest gas limit that allows the tx to execute successfully. + for lo+1 < hi { + mid := (hi + lo) / 2 + if mid > lo*2 { + // Most txs don't need much higher gas limit than their gas used, and most txs don't + // require near the full block limit of gas, so the selection of where to bisect the + // range here is skewed to favor the low side. + mid = lo * 2 + } + failed, _, err = execute(ctx, call, opts, mid) + if err != nil { + // This should not happen under normal conditions since if we make it this far the + // transaction had run without error at least once before. + log.Error("Execution error in estimate gas", "err", err) + return 0, nil, err + } + if failed { + lo = mid + } else { + hi = mid + } + } + return hi, nil, nil +} + +// execute is a helper that executes the transaction under a given gas limit and +// returns true if the transaction fails for a reason that might be related to +// not enough gas. A non-nil error means execution failed due to reasons unrelated +// to the gas limit. +func execute(ctx context.Context, call *core.Message, opts *Options, gasLimit uint64) (bool, *core.ExecutionResult, error) { + // Configure the call for this specific execution (and revert the change after) + defer func(gas uint64) { call.GasLimit = gas }(call.GasLimit) + call.GasLimit = gasLimit + + // Execute the call and separate execution faults caused by a lack of gas or + // other non-fixable conditions + result, err := run(ctx, call, opts) + if err != nil { + if errors.Is(err, core.ErrIntrinsicGas) { + return true, nil, nil // Special case, raise gas limit + } + return true, nil, err // Bail out + } + return result.Failed(), result, nil +} + +// run assembles the EVM as defined by the consensus rules and runs the requested +// call invocation. +func run(ctx context.Context, call *core.Message, opts *Options) (*core.ExecutionResult, error) { + // Assemble the call and the call context + var ( + msgContext = core.NewEVMTxContext(call) + evmContext = core.NewEVMBlockContext(opts.Header, opts.Chain, nil) + + dirtyState = opts.State.Copy() + evm = vm.NewEVM(evmContext, msgContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) + ) + // Monitor the outer context and interrupt the EVM upon cancellation. To avoid + // a dangling goroutine until the outer estimation finishes, create an internal + // context for the lifetime of this method call. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + <-ctx.Done() + evm.Cancel() + }() + // Execute the call, returning a wrapped error or the result + result, err := core.ApplyMessage(evm, call, new(core.GasPool).AddGas(math.MaxUint64)) + if vmerr := dirtyState.Error(); vmerr != nil { + return nil, vmerr + } + if err != nil { + return result, fmt.Errorf("failed with %d gas: %w", call.GasLimit, err) + } + return result, nil +} diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 89c132ddee..f322132769 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -40,6 +40,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/gasestimator" "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" @@ -1120,15 +1121,16 @@ func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash return doCall(ctx, b, args, state, header, overrides, blockOverrides, timeout, globalGasCap) } -func newRevertError(result *core.ExecutionResult) *revertError { - reason, errUnpack := abi.UnpackRevert(result.Revert()) - err := errors.New("execution reverted") +func newRevertError(revert []byte) *revertError { + err := vm.ErrExecutionReverted + + reason, errUnpack := abi.UnpackRevert(revert) if errUnpack == nil { - err = fmt.Errorf("execution reverted: %v", reason) + err = fmt.Errorf("%w: %v", vm.ErrExecutionReverted, reason) } return &revertError{ error: err, - reason: hexutil.Encode(result.Revert()), + reason: hexutil.Encode(revert), } } @@ -1167,147 +1169,44 @@ func (s *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockNrO } // If the result contains a revert reason, try to unpack and return it. if len(result.Revert()) > 0 { - return nil, newRevertError(result) + return nil, newRevertError(result.Revert()) } return result.Return(), result.Err } -// executeEstimate is a helper that executes the transaction under a given gas limit and returns -// true if the transaction fails for a reason that might be related to not enough gas. A non-nil -// error means execution failed due to reasons unrelated to the gas limit. -func executeEstimate(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, gasCap uint64, gasLimit uint64) (bool, *core.ExecutionResult, error) { - args.Gas = (*hexutil.Uint64)(&gasLimit) - result, err := doCall(ctx, b, args, state, header, nil, nil, 0, gasCap) - if err != nil { - if errors.Is(err, core.ErrIntrinsicGas) { - return true, nil, nil // Special case, raise gas limit - } - return true, nil, err // Bail out - } - return result.Failed(), result, nil -} - // DoEstimateGas returns the lowest possible gas limit that allows the transaction to run // successfully at block `blockNrOrHash`. It returns error if the transaction would revert, or if // there are unexpected failures. The gas limit is capped by both `args.Gas` (if non-nil & // non-zero) and `gasCap` (if non-zero). func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, gasCap uint64) (hexutil.Uint64, error) { - // Binary search the gas limit, as it may need to be higher than the amount used - var ( - lo uint64 // lowest-known gas limit where tx execution fails - hi uint64 // lowest-known gas limit where tx execution succeeds - ) - // Use zero address if sender unspecified. - if args.From == nil { - args.From = new(common.Address) - } - // Determine the highest gas limit can be used during the estimation. - if args.Gas != nil && uint64(*args.Gas) >= params.TxGas { - hi = uint64(*args.Gas) - } else { - // Retrieve the block to act as the gas ceiling - block, err := b.BlockByNumberOrHash(ctx, blockNrOrHash) - if err != nil { - return 0, err - } - if block == nil { - return 0, errors.New("block not found") - } - hi = block.GasLimit() - } - // Normalize the max fee per gas the call is willing to spend. - var feeCap *big.Int - if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { - return 0, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } else if args.GasPrice != nil { - feeCap = args.GasPrice.ToInt() - } else if args.MaxFeePerGas != nil { - feeCap = args.MaxFeePerGas.ToInt() - } else { - feeCap = common.Big0 - } - + // Retrieve the base state and mutate it with any overrides state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return 0, err } - if err := overrides.Apply(state); err != nil { + if err = overrides.Apply(state); err != nil { return 0, err } - - // Recap the highest gas limit with account's available balance. - if feeCap.BitLen() != 0 { - balance := state.GetBalance(*args.From) // from can't be nil - available := new(big.Int).Set(balance) - if args.Value != nil { - if args.Value.ToInt().Cmp(available) >= 0 { - return 0, core.ErrInsufficientFundsForTransfer - } - available.Sub(available, args.Value.ToInt()) - } - allowance := new(big.Int).Div(available, feeCap) - - // If the allowance is larger than maximum uint64, skip checking - if allowance.IsUint64() && hi > allowance.Uint64() { - transfer := args.Value - if transfer == nil { - transfer = new(hexutil.Big) - } - log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, - "sent", transfer.ToInt(), "maxFeePerGas", feeCap, "fundable", allowance) - hi = allowance.Uint64() - } + // Construct the gas estimator option from the user input + opts := &gasestimator.Options{ + Config: b.ChainConfig(), + Chain: NewChainContext(ctx, b), + Header: header, + State: state, } - // Recap the highest gas allowance with specified gascap. - if gasCap != 0 && hi > gasCap { - log.Warn("Caller gas above allowance, capping", "requested", hi, "cap", gasCap) - hi = gasCap - } - - // We first execute the transaction at the highest allowable gas limit, since if this fails we - // can return error immediately. - failed, result, err := executeEstimate(ctx, b, args, state.Copy(), header, gasCap, hi) + // Run the gas estimation andwrap any revertals into a custom return + call, err := args.ToMessage(gasCap, header.BaseFee) if err != nil { return 0, err } - if failed { - if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) { - if len(result.Revert()) > 0 { - return 0, newRevertError(result) - } - return 0, result.Err + estimate, revert, err := gasestimator.Estimate(ctx, call, opts, gasCap) + if err != nil { + if len(revert) > 0 { + return 0, newRevertError(revert) } - return 0, fmt.Errorf("gas required exceeds allowance (%d)", hi) + return 0, err } - // For almost any transaction, the gas consumed by the unconstrained execution above - // lower-bounds the gas limit required for it to succeed. One exception is those txs that - // explicitly check gas remaining in order to successfully execute within a given limit, but we - // probably don't want to return a lowest possible gas limit for these cases anyway. - lo = result.UsedGas - 1 - - // Binary search for the smallest gas limit that allows the tx to execute successfully. - for lo+1 < hi { - mid := (hi + lo) / 2 - if mid > lo*2 { - // Most txs don't need much higher gas limit than their gas used, and most txs don't - // require near the full block limit of gas, so the selection of where to bisect the - // range here is skewed to favor the low side. - mid = lo * 2 - } - failed, _, err = executeEstimate(ctx, b, args, state.Copy(), header, gasCap, mid) - if err != nil { - // This should not happen under normal conditions since if we make it this far the - // transaction had run without error at least once before. - log.Error("execution error in estimate gas", "err", err) - return 0, err - } - if failed { - lo = mid - } else { - hi = mid - } - } - return hexutil.Uint64(hi), nil + return hexutil.Uint64(estimate), nil } // EstimateGas returns the lowest possible gas limit that allows the transaction to run diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 4a7694737e..9b08fd8d42 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -910,18 +910,18 @@ func TestCall(t *testing.T) { } } -type Account struct { +type account struct { key *ecdsa.PrivateKey addr common.Address } -func newAccounts(n int) (accounts []Account) { +func newAccounts(n int) (accounts []account) { for i := 0; i < n; i++ { key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - accounts = append(accounts, Account{key: key, addr: addr}) + accounts = append(accounts, account{key: key, addr: addr}) } - slices.SortFunc(accounts, func(a, b Account) int { return a.addr.Cmp(b.addr) }) + slices.SortFunc(accounts, func(a, b account) int { return a.addr.Cmp(b.addr) }) return accounts } From 5b57727d6de25493245ead838412188ffb6f1324 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 27 Nov 2023 15:39:28 +0100 Subject: [PATCH 04/43] go.mod: update uint256 to v1.2.4 (#28612) --- go.mod | 2 +- go.sum | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 32cfe26b14..042d7155be 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/hashicorp/go-bexpr v0.1.10 github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 github.com/holiman/bloomfilter/v2 v2.0.3 - github.com/holiman/uint256 v1.2.3 + github.com/holiman/uint256 v1.2.4 github.com/huin/goupnp v1.3.0 github.com/influxdata/influxdb-client-go/v2 v2.4.0 github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c diff --git a/go.sum b/go.sum index e62d7d36ab..d5e7901e5a 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= -github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= @@ -147,8 +145,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20230914135612-d1b03fcb8e58 h1:PwUlswsGOrLB677lW4XrlWLeszY3BaDGbvZ6dYk28tQ= -github.com/crate-crypto/go-ipa v0.0.0-20230914135612-d1b03fcb8e58/go.mod h1:J+gsi6D4peY0kyhaklyXFRVHOQWI2I5uU0c2+/90HYc= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= @@ -205,8 +201,6 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILD github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-verkle v0.1.1-0.20231004173727-0a4e93ed640b h1:LHeiiSTL2FEGCP1ov6FqkikiViqygeVo1ZwJ1x3nYSE= -github.com/gballet/go-verkle v0.1.1-0.20231004173727-0a4e93ed640b/go.mod h1:7JamHhSTnnHDhcI3G8r4sWaD9XlleriqVlC3FeAQJKM= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -347,8 +341,8 @@ github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZ github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= -github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.4 h1:jUc4Nk8fm9jZabQuqr2JzednajVmBpC+oiTiXZJEApU= +github.com/holiman/uint256 v1.2.4/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= From 71817f318edc8c37161f38aa7a0407d4efc2429f Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 28 Nov 2023 15:38:30 +0800 Subject: [PATCH 05/43] eth/catalyst, eth/downloader: expose more sync information (#28584) This change exposes more information from sync module internally --- eth/catalyst/api.go | 7 ++--- eth/downloader/skeleton.go | 46 ++++++++++++++++----------------- eth/downloader/skeleton_test.go | 12 ++++----- trie/verkle_test.go | 8 +++--- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index d1e1991414..37b0248f28 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -611,7 +611,8 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS // Although we don't want to trigger a sync, if there is one already in // progress, try to extend if with the current payload request to relieve // some strain from the forkchoice update. - if err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()); err == nil { + err := api.eth.Downloader().BeaconExtend(api.eth.SyncMode(), block.Header()) + if err == nil { log.Debug("Payload accepted for sync extension", "number", block.NumberU64(), "hash", block.Hash()) return engine.PayloadStatusV1{Status: engine.SYNCING}, nil } @@ -623,12 +624,12 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (engine.PayloadS // In full sync mode, failure to import a well-formed block can only mean // that the parent state is missing and the syncer rejected extending the // current cycle with the new payload. - log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash(), "reason", err) } else { // In non-full sync mode (i.e. snap sync) all payloads are rejected until // snap sync terminates as snap sync relies on direct database injections // and cannot afford concurrent out-if-band modifications via imports. - log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) + log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash(), "reason", err) } return engine.PayloadStatusV1{Status: engine.SYNCING}, nil } diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index 4f1f462048..f40ca24d99 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -69,9 +69,17 @@ var errSyncReorged = errors.New("sync reorged") // might still be propagating. var errTerminated = errors.New("terminated") -// errReorgDenied is returned if an attempt is made to extend the beacon chain -// with a new header, but it does not link up to the existing sync. -var errReorgDenied = errors.New("non-forced head reorg denied") +// errChainReorged is an internal helper error to signal that the header chain +// of the current sync cycle was (partially) reorged. +var errChainReorged = errors.New("chain reorged") + +// errChainGapped is an internal helper error to signal that the header chain +// of the current sync cycle is gaped with the one advertised by consensus client. +var errChainGapped = errors.New("chain gapped") + +// errChainForked is an internal helper error to signal that the header chain +// of the current sync cycle is forked with the one advertised by consensus client. +var errChainForked = errors.New("chain forked") func init() { // Tuning parameters is nice, but the scratch space must be assignable in @@ -271,9 +279,9 @@ func (s *skeleton) startup() { newhead, err := s.sync(head) switch { case err == errSyncLinked: - // Sync cycle linked up to the genesis block. Tear down the loop - // and restart it so, it can properly notify the backfiller. Don't - // account a new head. + // Sync cycle linked up to the genesis block, or the existent chain + // segment. Tear down the loop and restart it so, it can properly + // notify the backfiller. Don't account a new head. head = nil case err == errSyncMerged: @@ -457,15 +465,16 @@ func (s *skeleton) sync(head *types.Header) (*types.Header, error) { // we don't seamlessly integrate reorgs to keep things simple. If the // network starts doing many mini reorgs, it might be worthwhile handling // a limited depth without an error. - if reorged := s.processNewHead(event.header, event.final, event.force); reorged { + if err := s.processNewHead(event.header, event.final); err != nil { // If a reorg is needed, and we're forcing the new head, signal // the syncer to tear down and start over. Otherwise, drop the // non-force reorg. if event.force { event.errc <- nil // forced head reorg accepted + log.Info("Restarting sync cycle", "reason", err) return event.header, errSyncReorged } - event.errc <- errReorgDenied + event.errc <- err continue } event.errc <- nil // head extension accepted @@ -610,7 +619,7 @@ func (s *skeleton) saveSyncStatus(db ethdb.KeyValueWriter) { // accepts and integrates it into the skeleton or requests a reorg. Upon reorg, // the syncer will tear itself down and restart with a fresh head. It is simpler // to reconstruct the sync state than to mutate it and hope for the best. -func (s *skeleton) processNewHead(head *types.Header, final *types.Header, force bool) bool { +func (s *skeleton) processNewHead(head *types.Header, final *types.Header) error { // If a new finalized block was announced, update the sync process independent // of what happens with the sync head below if final != nil { @@ -631,26 +640,17 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header, force // once more, ignore it instead of tearing down sync for a noop. if lastchain.Head == lastchain.Tail { if current := rawdb.ReadSkeletonHeader(s.db, number); current.Hash() == head.Hash() { - return false + return nil } } // Not a noop / double head announce, abort with a reorg - if force { - log.Warn("Beacon chain reorged", "tail", lastchain.Tail, "head", lastchain.Head, "newHead", number) - } - return true + return fmt.Errorf("%w, tail: %d, head: %d, newHead: %d", errChainReorged, lastchain.Tail, lastchain.Head, number) } if lastchain.Head+1 < number { - if force { - log.Warn("Beacon chain gapped", "head", lastchain.Head, "newHead", number) - } - return true + return fmt.Errorf("%w, head: %d, newHead: %d", errChainGapped, lastchain.Head, number) } if parent := rawdb.ReadSkeletonHeader(s.db, number-1); parent.Hash() != head.ParentHash { - if force { - log.Warn("Beacon chain forked", "ancestor", number-1, "hash", parent.Hash(), "want", head.ParentHash) - } - return true + return fmt.Errorf("%w, ancestor: %d, hash: %s, want: %s", errChainForked, number-1, parent.Hash(), head.ParentHash) } // New header seems to be in the last subchain range. Unwind any extra headers // from the chain tip and insert the new head. We won't delete any trimmed @@ -666,7 +666,7 @@ func (s *skeleton) processNewHead(head *types.Header, final *types.Header, force if err := batch.Write(); err != nil { log.Crit("Failed to write skeleton sync status", "err", err) } - return false + return nil } // assignTasks attempts to match idle peers to pending header retrievals. diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index c31007765a..aceadd00d3 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -434,7 +434,7 @@ func TestSkeletonSyncExtend(t *testing.T) { newstate: []*subchain{ {Head: 49, Tail: 49}, }, - err: errReorgDenied, + err: errChainReorged, }, // Initialize a sync and try to extend it with a number-wise sequential // header, but a hash wise non-linking one. @@ -444,7 +444,7 @@ func TestSkeletonSyncExtend(t *testing.T) { newstate: []*subchain{ {Head: 49, Tail: 49}, }, - err: errReorgDenied, + err: errChainForked, }, // Initialize a sync and try to extend it with a non-linking future block. { @@ -453,7 +453,7 @@ func TestSkeletonSyncExtend(t *testing.T) { newstate: []*subchain{ {Head: 49, Tail: 49}, }, - err: errReorgDenied, + err: errChainGapped, }, // Initialize a sync and try to extend it with a past canonical block. { @@ -462,7 +462,7 @@ func TestSkeletonSyncExtend(t *testing.T) { newstate: []*subchain{ {Head: 50, Tail: 50}, }, - err: errReorgDenied, + err: errChainReorged, }, // Initialize a sync and try to extend it with a past sidechain block. { @@ -471,7 +471,7 @@ func TestSkeletonSyncExtend(t *testing.T) { newstate: []*subchain{ {Head: 50, Tail: 50}, }, - err: errReorgDenied, + err: errChainReorged, }, } for i, tt := range tests { @@ -487,7 +487,7 @@ func TestSkeletonSyncExtend(t *testing.T) { skeleton.Sync(tt.head, nil, true) <-wait - if err := skeleton.Sync(tt.extend, nil, false); err != tt.err { + if err := skeleton.Sync(tt.extend, nil, false); !errors.Is(err, tt.err) { t.Errorf("test %d: extension failure mismatch: have %v, want %v", i, err, tt.err) } skeleton.Terminate() diff --git a/trie/verkle_test.go b/trie/verkle_test.go index 44fb7dc29e..bd31ea3879 100644 --- a/trie/verkle_test.go +++ b/trie/verkle_test.go @@ -31,24 +31,24 @@ import ( var ( accounts = map[common.Address]*types.StateAccount{ - common.Address{1}: { + {1}: { Nonce: 100, Balance: big.NewInt(100), CodeHash: common.Hash{0x1}.Bytes(), }, - common.Address{2}: { + {2}: { Nonce: 200, Balance: big.NewInt(200), CodeHash: common.Hash{0x2}.Bytes(), }, } storages = map[common.Address]map[common.Hash][]byte{ - common.Address{1}: { + {1}: { common.Hash{10}: []byte{10}, common.Hash{11}: []byte{11}, common.MaxHash: []byte{0xff}, }, - common.Address{2}: { + {2}: { common.Hash{20}: []byte{20}, common.Hash{21}: []byte{21}, common.MaxHash: []byte{0xff}, From 58297e339b26d09a0c21e550ee4b6ed6205cedcd Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 28 Nov 2023 01:01:23 -0700 Subject: [PATCH 06/43] light: remove package light(#28614) This changes removes the package 'light', which is currently unused. --- light/lightchain.go | 531 ------------------------------------- light/lightchain_test.go | 358 ------------------------- light/odr.go | 196 -------------- light/odr_test.go | 339 ------------------------ light/odr_util.go | 275 ------------------- light/postprocess.go | 538 ------------------------------------- light/trie.go | 319 ---------------------- light/trie_test.go | 95 ------- light/txpool.go | 556 --------------------------------------- light/txpool_test.go | 147 ----------- 10 files changed, 3354 deletions(-) delete mode 100644 light/lightchain.go delete mode 100644 light/lightchain_test.go delete mode 100644 light/odr.go delete mode 100644 light/odr_test.go delete mode 100644 light/odr_util.go delete mode 100644 light/postprocess.go delete mode 100644 light/trie.go delete mode 100644 light/trie_test.go delete mode 100644 light/txpool.go delete mode 100644 light/txpool_test.go diff --git a/light/lightchain.go b/light/lightchain.go deleted file mode 100644 index 617658b85b..0000000000 --- a/light/lightchain.go +++ /dev/null @@ -1,531 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package light implements on-demand retrieval capable state and chain objects -// for the Ethereum Light Client. -package light - -import ( - "context" - "errors" - "math/big" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" -) - -var ( - bodyCacheLimit = 256 - blockCacheLimit = 256 -) - -// LightChain represents a canonical chain that by default only handles block -// headers, downloading block bodies and receipts on demand through an ODR -// interface. It only does header validation during chain insertion. -type LightChain struct { - hc *core.HeaderChain - indexerConfig *IndexerConfig - chainDb ethdb.Database - engine consensus.Engine - odr OdrBackend - chainFeed event.Feed - chainSideFeed event.Feed - chainHeadFeed event.Feed - scope event.SubscriptionScope - genesisBlock *types.Block - forker *core.ForkChoice - - bodyCache *lru.Cache[common.Hash, *types.Body] - bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] - blockCache *lru.Cache[common.Hash, *types.Block] - - chainmu sync.RWMutex // protects header inserts - quit chan struct{} - wg sync.WaitGroup - - // Atomic boolean switches: - stopped atomic.Bool // whether LightChain is stopped or running - procInterrupt atomic.Bool // interrupts chain insert -} - -// NewLightChain returns a fully initialised light chain using information -// available in the database. It initialises the default Ethereum header -// validator. -func NewLightChain(odr OdrBackend, config *params.ChainConfig, engine consensus.Engine) (*LightChain, error) { - bc := &LightChain{ - chainDb: odr.Database(), - indexerConfig: odr.IndexerConfig(), - odr: odr, - quit: make(chan struct{}), - bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), - bodyRLPCache: lru.NewCache[common.Hash, rlp.RawValue](bodyCacheLimit), - blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit), - engine: engine, - } - bc.forker = core.NewForkChoice(bc, nil) - var err error - bc.hc, err = core.NewHeaderChain(odr.Database(), config, bc.engine, bc.getProcInterrupt) - if err != nil { - return nil, err - } - bc.genesisBlock, _ = bc.GetBlockByNumber(NoOdr, 0) - if bc.genesisBlock == nil { - return nil, core.ErrNoGenesis - } - if err := bc.loadLastState(); err != nil { - return nil, err - } - // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain - for hash := range core.BadHashes { - if header := bc.GetHeaderByHash(hash); header != nil { - log.Error("Found bad hash, rewinding chain", "number", header.Number, "hash", header.ParentHash) - bc.SetHead(header.Number.Uint64() - 1) - log.Info("Chain rewind was successful, resuming normal operation") - } - } - return bc, nil -} - -func (lc *LightChain) getProcInterrupt() bool { - return lc.procInterrupt.Load() -} - -// Odr returns the ODR backend of the chain -func (lc *LightChain) Odr() OdrBackend { - return lc.odr -} - -// HeaderChain returns the underlying header chain. -func (lc *LightChain) HeaderChain() *core.HeaderChain { - return lc.hc -} - -// loadLastState loads the last known chain state from the database. This method -// assumes that the chain manager mutex is held. -func (lc *LightChain) loadLastState() error { - if head := rawdb.ReadHeadHeaderHash(lc.chainDb); head == (common.Hash{}) { - // Corrupt or empty database, init from scratch - lc.Reset() - } else { - header := lc.GetHeaderByHash(head) - if header == nil { - // Corrupt or empty database, init from scratch - lc.Reset() - } else { - lc.hc.SetCurrentHeader(header) - } - } - // Issue a status log and return - header := lc.hc.CurrentHeader() - headerTd := lc.GetTd(header.Hash(), header.Number.Uint64()) - log.Info("Loaded most recent local header", "number", header.Number, "hash", header.Hash(), "td", headerTd, "age", common.PrettyAge(time.Unix(int64(header.Time), 0))) - return nil -} - -// SetHead rewinds the local chain to a new head. Everything above the new -// head will be deleted and the new one set. -func (lc *LightChain) SetHead(head uint64) error { - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - lc.hc.SetHead(head, nil, nil) - return lc.loadLastState() -} - -// SetHeadWithTimestamp rewinds the local chain to a new head that has at max -// the given timestamp. Everything above the new head will be deleted and the -// new one set. -func (lc *LightChain) SetHeadWithTimestamp(timestamp uint64) error { - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - lc.hc.SetHeadWithTimestamp(timestamp, nil, nil) - return lc.loadLastState() -} - -// GasLimit returns the gas limit of the current HEAD block. -func (lc *LightChain) GasLimit() uint64 { - return lc.hc.CurrentHeader().GasLimit -} - -// Reset purges the entire blockchain, restoring it to its genesis state. -func (lc *LightChain) Reset() { - lc.ResetWithGenesisBlock(lc.genesisBlock) -} - -// ResetWithGenesisBlock purges the entire blockchain, restoring it to the -// specified genesis state. -func (lc *LightChain) ResetWithGenesisBlock(genesis *types.Block) { - // Dump the entire block chain and purge the caches - lc.SetHead(0) - - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - // Prepare the genesis block and reinitialise the chain - batch := lc.chainDb.NewBatch() - rawdb.WriteTd(batch, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) - rawdb.WriteBlock(batch, genesis) - rawdb.WriteHeadHeaderHash(batch, genesis.Hash()) - if err := batch.Write(); err != nil { - log.Crit("Failed to reset genesis block", "err", err) - } - lc.genesisBlock = genesis - lc.hc.SetGenesis(lc.genesisBlock.Header()) - lc.hc.SetCurrentHeader(lc.genesisBlock.Header()) -} - -// Accessors - -// Engine retrieves the light chain's consensus engine. -func (lc *LightChain) Engine() consensus.Engine { return lc.engine } - -// Genesis returns the genesis block -func (lc *LightChain) Genesis() *types.Block { - return lc.genesisBlock -} - -func (lc *LightChain) StateCache() state.Database { - panic("not implemented") -} - -// GetBody retrieves a block body (transactions and uncles) from the database -// or ODR service by hash, caching it if found. -func (lc *LightChain) GetBody(ctx context.Context, hash common.Hash) (*types.Body, error) { - // Short circuit if the body's already in the cache, retrieve otherwise - if cached, ok := lc.bodyCache.Get(hash); ok { - return cached, nil - } - number := lc.hc.GetBlockNumber(hash) - if number == nil { - return nil, errors.New("unknown block") - } - body, err := GetBody(ctx, lc.odr, hash, *number) - if err != nil { - return nil, err - } - // Cache the found body for next time and return - lc.bodyCache.Add(hash, body) - return body, nil -} - -// GetBodyRLP retrieves a block body in RLP encoding from the database or -// ODR service by hash, caching it if found. -func (lc *LightChain) GetBodyRLP(ctx context.Context, hash common.Hash) (rlp.RawValue, error) { - // Short circuit if the body's already in the cache, retrieve otherwise - if cached, ok := lc.bodyRLPCache.Get(hash); ok { - return cached, nil - } - number := lc.hc.GetBlockNumber(hash) - if number == nil { - return nil, errors.New("unknown block") - } - body, err := GetBodyRLP(ctx, lc.odr, hash, *number) - if err != nil { - return nil, err - } - // Cache the found body for next time and return - lc.bodyRLPCache.Add(hash, body) - return body, nil -} - -// HasBlock checks if a block is fully present in the database or not, caching -// it if present. -func (lc *LightChain) HasBlock(hash common.Hash, number uint64) bool { - blk, _ := lc.GetBlock(NoOdr, hash, number) - return blk != nil -} - -// GetBlock retrieves a block from the database or ODR service by hash and number, -// caching it if found. -func (lc *LightChain) GetBlock(ctx context.Context, hash common.Hash, number uint64) (*types.Block, error) { - // Short circuit if the block's already in the cache, retrieve otherwise - if block, ok := lc.blockCache.Get(hash); ok { - return block, nil - } - block, err := GetBlock(ctx, lc.odr, hash, number) - if err != nil { - return nil, err - } - // Cache the found block for next time and return - lc.blockCache.Add(block.Hash(), block) - return block, nil -} - -// GetBlockByHash retrieves a block from the database or ODR service by hash, -// caching it if found. -func (lc *LightChain) GetBlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - number := lc.hc.GetBlockNumber(hash) - if number == nil { - return nil, errors.New("unknown block") - } - return lc.GetBlock(ctx, hash, *number) -} - -// GetBlockByNumber retrieves a block from the database or ODR service by -// number, caching it (associated with its hash) if found. -func (lc *LightChain) GetBlockByNumber(ctx context.Context, number uint64) (*types.Block, error) { - hash, err := GetCanonicalHash(ctx, lc.odr, number) - if hash == (common.Hash{}) || err != nil { - return nil, err - } - return lc.GetBlock(ctx, hash, number) -} - -// Stop stops the blockchain service. If any imports are currently in progress -// it will abort them using the procInterrupt. -func (lc *LightChain) Stop() { - if !lc.stopped.CompareAndSwap(false, true) { - return - } - close(lc.quit) - lc.StopInsert() - lc.wg.Wait() - log.Info("Blockchain stopped") -} - -// StopInsert interrupts all insertion methods, causing them to return -// errInsertionInterrupted as soon as possible. Insertion is permanently disabled after -// calling this method. -func (lc *LightChain) StopInsert() { - lc.procInterrupt.Store(true) -} - -// Rollback is designed to remove a chain of links from the database that aren't -// certain enough to be valid. -func (lc *LightChain) Rollback(chain []common.Hash) { - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - batch := lc.chainDb.NewBatch() - for i := len(chain) - 1; i >= 0; i-- { - hash := chain[i] - - // Degrade the chain markers if they are explicitly reverted. - // In theory we should update all in-memory markers in the - // last step, however the direction of rollback is from high - // to low, so it's safe the update in-memory markers directly. - if head := lc.hc.CurrentHeader(); head.Hash() == hash { - rawdb.WriteHeadHeaderHash(batch, head.ParentHash) - lc.hc.SetCurrentHeader(lc.GetHeader(head.ParentHash, head.Number.Uint64()-1)) - } - } - if err := batch.Write(); err != nil { - log.Crit("Failed to rollback light chain", "error", err) - } -} - -func (lc *LightChain) InsertHeader(header *types.Header) error { - // Verify the header first before obtaining the lock - headers := []*types.Header{header} - if _, err := lc.hc.ValidateHeaderChain(headers); err != nil { - return err - } - // Make sure only one thread manipulates the chain at once - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - lc.wg.Add(1) - defer lc.wg.Done() - - _, err := lc.hc.WriteHeaders(headers) - log.Info("Inserted header", "number", header.Number, "hash", header.Hash()) - return err -} - -func (lc *LightChain) SetCanonical(header *types.Header) error { - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - lc.wg.Add(1) - defer lc.wg.Done() - - if err := lc.hc.Reorg([]*types.Header{header}); err != nil { - return err - } - // Emit events - block := types.NewBlockWithHeader(header) - lc.chainFeed.Send(core.ChainEvent{Block: block, Hash: block.Hash()}) - lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: block}) - log.Info("Set the chain head", "number", block.Number(), "hash", block.Hash()) - return nil -} - -// InsertHeaderChain attempts to insert the given header chain in to the local -// chain, possibly creating a reorg. If an error is returned, it will return the -// index number of the failing header as well an error describing what went wrong. - -// In the case of a light chain, InsertHeaderChain also creates and posts light -// chain events when necessary. -func (lc *LightChain) InsertHeaderChain(chain []*types.Header) (int, error) { - if len(chain) == 0 { - return 0, nil - } - start := time.Now() - if i, err := lc.hc.ValidateHeaderChain(chain); err != nil { - return i, err - } - - // Make sure only one thread manipulates the chain at once - lc.chainmu.Lock() - defer lc.chainmu.Unlock() - - lc.wg.Add(1) - defer lc.wg.Done() - - status, err := lc.hc.InsertHeaderChain(chain, start, lc.forker) - if err != nil || len(chain) == 0 { - return 0, err - } - - // Create chain event for the new head block of this insertion. - var ( - lastHeader = chain[len(chain)-1] - block = types.NewBlockWithHeader(lastHeader) - ) - switch status { - case core.CanonStatTy: - lc.chainFeed.Send(core.ChainEvent{Block: block, Hash: block.Hash()}) - lc.chainHeadFeed.Send(core.ChainHeadEvent{Block: block}) - case core.SideStatTy: - lc.chainSideFeed.Send(core.ChainSideEvent{Block: block}) - } - return 0, err -} - -// CurrentHeader retrieves the current head header of the canonical chain. The -// header is retrieved from the HeaderChain's internal cache. -func (lc *LightChain) CurrentHeader() *types.Header { - return lc.hc.CurrentHeader() -} - -// GetTd retrieves a block's total difficulty in the canonical chain from the -// database by hash and number, caching it if found. -func (lc *LightChain) GetTd(hash common.Hash, number uint64) *big.Int { - return lc.hc.GetTd(hash, number) -} - -// GetTdOdr retrieves the total difficult from the database or -// network by hash and number, caching it (associated with its hash) if found. -func (lc *LightChain) GetTdOdr(ctx context.Context, hash common.Hash, number uint64) *big.Int { - td := lc.GetTd(hash, number) - if td != nil { - return td - } - td, _ = GetTd(ctx, lc.odr, hash, number) - return td -} - -// GetHeader retrieves a block header from the database by hash and number, -// caching it if found. -func (lc *LightChain) GetHeader(hash common.Hash, number uint64) *types.Header { - return lc.hc.GetHeader(hash, number) -} - -// GetHeaderByHash retrieves a block header from the database by hash, caching it if -// found. -func (lc *LightChain) GetHeaderByHash(hash common.Hash) *types.Header { - return lc.hc.GetHeaderByHash(hash) -} - -// HasHeader checks if a block header is present in the database or not, caching -// it if present. -func (lc *LightChain) HasHeader(hash common.Hash, number uint64) bool { - return lc.hc.HasHeader(hash, number) -} - -// GetCanonicalHash returns the canonical hash for a given block number -func (bc *LightChain) GetCanonicalHash(number uint64) common.Hash { - return bc.hc.GetCanonicalHash(number) -} - -// GetAncestor retrieves the Nth ancestor of a given block. It assumes that either the given block or -// a close ancestor of it is canonical. maxNonCanonical points to a downwards counter limiting the -// number of blocks to be individually checked before we reach the canonical chain. -// -// Note: ancestor == 0 returns the same block, 1 returns its parent and so on. -func (lc *LightChain) GetAncestor(hash common.Hash, number, ancestor uint64, maxNonCanonical *uint64) (common.Hash, uint64) { - return lc.hc.GetAncestor(hash, number, ancestor, maxNonCanonical) -} - -// GetHeaderByNumber retrieves a block header from the database by number, -// caching it (associated with its hash) if found. -func (lc *LightChain) GetHeaderByNumber(number uint64) *types.Header { - return lc.hc.GetHeaderByNumber(number) -} - -// GetHeaderByNumberOdr retrieves a block header from the database or network -// by number, caching it (associated with its hash) if found. -func (lc *LightChain) GetHeaderByNumberOdr(ctx context.Context, number uint64) (*types.Header, error) { - if header := lc.hc.GetHeaderByNumber(number); header != nil { - return header, nil - } - return GetHeaderByNumber(ctx, lc.odr, number) -} - -// Config retrieves the header chain's chain configuration. -func (lc *LightChain) Config() *params.ChainConfig { return lc.hc.Config() } - -// LockChain locks the chain mutex for reading so that multiple canonical hashes can be -// retrieved while it is guaranteed that they belong to the same version of the chain -func (lc *LightChain) LockChain() { - lc.chainmu.RLock() -} - -// UnlockChain unlocks the chain mutex -func (lc *LightChain) UnlockChain() { - lc.chainmu.RUnlock() -} - -// SubscribeChainEvent registers a subscription of ChainEvent. -func (lc *LightChain) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { - return lc.scope.Track(lc.chainFeed.Subscribe(ch)) -} - -// SubscribeChainHeadEvent registers a subscription of ChainHeadEvent. -func (lc *LightChain) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { - return lc.scope.Track(lc.chainHeadFeed.Subscribe(ch)) -} - -// SubscribeChainSideEvent registers a subscription of ChainSideEvent. -func (lc *LightChain) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { - return lc.scope.Track(lc.chainSideFeed.Subscribe(ch)) -} - -// SubscribeLogsEvent implements the interface of filters.Backend -// LightChain does not send logs events, so return an empty subscription. -func (lc *LightChain) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { - return lc.scope.Track(new(event.Feed).Subscribe(ch)) -} - -// SubscribeRemovedLogsEvent implements the interface of filters.Backend -// LightChain does not send core.RemovedLogsEvent, so return an empty subscription. -func (lc *LightChain) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { - return lc.scope.Track(new(event.Feed).Subscribe(ch)) -} diff --git a/light/lightchain_test.go b/light/lightchain_test.go deleted file mode 100644 index 5694ca72c2..0000000000 --- a/light/lightchain_test.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "errors" - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -// So we can deterministically seed different blockchains -var ( - canonicalSeed = 1 - forkSeed = 2 -) - -// makeHeaderChain creates a deterministic chain of headers rooted at parent. -func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) []*types.Header { - blocks, _ := core.GenerateChain(params.TestChainConfig, types.NewBlockWithHeader(parent), ethash.NewFaker(), db, n, func(i int, b *core.BlockGen) { - b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)}) - }) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - return headers -} - -// newCanonical creates a chain database, and injects a deterministic canonical -// chain. Depending on the full flag, if creates either a full block chain or a -// header only chain. -func newCanonical(n int) (ethdb.Database, *LightChain, error) { - db := rawdb.NewMemoryDatabase() - gspec := core.Genesis{Config: params.TestChainConfig} - genesis := gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) - blockchain, _ := NewLightChain(&dummyOdr{db: db, indexerConfig: TestClientIndexerConfig}, gspec.Config, ethash.NewFaker()) - - // Create and inject the requested chain - if n == 0 { - return db, blockchain, nil - } - // Header-only chain requested - headers := makeHeaderChain(genesis.Header(), n, db, canonicalSeed) - _, err := blockchain.InsertHeaderChain(headers) - return db, blockchain, err -} - -// newTestLightChain creates a LightChain that doesn't validate anything. -func newTestLightChain() *LightChain { - db := rawdb.NewMemoryDatabase() - gspec := &core.Genesis{ - Difficulty: big.NewInt(1), - Config: params.TestChainConfig, - } - gspec.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)) - lc, err := NewLightChain(&dummyOdr{db: db}, gspec.Config, ethash.NewFullFaker()) - if err != nil { - panic(err) - } - return lc -} - -// Test fork of length N starting from block i -func testFork(t *testing.T, LightChain *LightChain, i, n int, comparator func(td1, td2 *big.Int)) { - // Copy old chain up to #i into a new db - db, LightChain2, err := newCanonical(i) - if err != nil { - t.Fatal("could not make new canonical in testFork", err) - } - // Assert the chains have the same header/block at #i - var hash1, hash2 common.Hash - hash1 = LightChain.GetHeaderByNumber(uint64(i)).Hash() - hash2 = LightChain2.GetHeaderByNumber(uint64(i)).Hash() - if hash1 != hash2 { - t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", i, hash2, hash1) - } - // Extend the newly created chain - headerChainB := makeHeaderChain(LightChain2.CurrentHeader(), n, db, forkSeed) - if _, err := LightChain2.InsertHeaderChain(headerChainB); err != nil { - t.Fatalf("failed to insert forking chain: %v", err) - } - // Sanity check that the forked chain can be imported into the original - var tdPre, tdPost *big.Int - cur := LightChain.CurrentHeader() - tdPre = LightChain.GetTd(cur.Hash(), cur.Number.Uint64()) - if err := testHeaderChainImport(headerChainB, LightChain); err != nil { - t.Fatalf("failed to import forked header chain: %v", err) - } - last := headerChainB[len(headerChainB)-1] - tdPost = LightChain.GetTd(last.Hash(), last.Number.Uint64()) - // Compare the total difficulties of the chains - comparator(tdPre, tdPost) -} - -// testHeaderChainImport tries to process a chain of header, writing them into -// the database if successful. -func testHeaderChainImport(chain []*types.Header, lightchain *LightChain) error { - for _, header := range chain { - // Try and validate the header - if err := lightchain.engine.VerifyHeader(lightchain.hc, header); err != nil { - return err - } - // Manually insert the header into the database, but don't reorganize (allows subsequent testing) - lightchain.chainmu.Lock() - rawdb.WriteTd(lightchain.chainDb, header.Hash(), header.Number.Uint64(), - new(big.Int).Add(header.Difficulty, lightchain.GetTd(header.ParentHash, header.Number.Uint64()-1))) - rawdb.WriteHeader(lightchain.chainDb, header) - lightchain.chainmu.Unlock() - } - return nil -} - -// Tests that given a starting canonical chain of a given size, it can be extended -// with various length chains. -func TestExtendCanonicalHeaders(t *testing.T) { - length := 5 - - // Make first chain starting from genesis - _, processor, err := newCanonical(length) - if err != nil { - t.Fatalf("failed to make new canonical chain: %v", err) - } - // Define the difficulty comparator - better := func(td1, td2 *big.Int) { - if td2.Cmp(td1) <= 0 { - t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1) - } - } - // Start fork from current height - testFork(t, processor, length, 1, better) - testFork(t, processor, length, 2, better) - testFork(t, processor, length, 5, better) - testFork(t, processor, length, 10, better) -} - -// Tests that given a starting canonical chain of a given size, creating shorter -// forks do not take canonical ownership. -func TestShorterForkHeaders(t *testing.T) { - length := 10 - - // Make first chain starting from genesis - _, processor, err := newCanonical(length) - if err != nil { - t.Fatalf("failed to make new canonical chain: %v", err) - } - // Define the difficulty comparator - worse := func(td1, td2 *big.Int) { - if td2.Cmp(td1) >= 0 { - t.Errorf("total difficulty mismatch: have %v, expected less than %v", td2, td1) - } - } - // Sum of numbers must be less than `length` for this to be a shorter fork - testFork(t, processor, 0, 3, worse) - testFork(t, processor, 0, 7, worse) - testFork(t, processor, 1, 1, worse) - testFork(t, processor, 1, 7, worse) - testFork(t, processor, 5, 3, worse) - testFork(t, processor, 5, 4, worse) -} - -// Tests that given a starting canonical chain of a given size, creating longer -// forks do take canonical ownership. -func TestLongerForkHeaders(t *testing.T) { - length := 10 - - // Make first chain starting from genesis - _, processor, err := newCanonical(length) - if err != nil { - t.Fatalf("failed to make new canonical chain: %v", err) - } - // Define the difficulty comparator - better := func(td1, td2 *big.Int) { - if td2.Cmp(td1) <= 0 { - t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1) - } - } - // Sum of numbers must be greater than `length` for this to be a longer fork - testFork(t, processor, 0, 11, better) - testFork(t, processor, 0, 15, better) - testFork(t, processor, 1, 10, better) - testFork(t, processor, 1, 12, better) - testFork(t, processor, 5, 6, better) - testFork(t, processor, 5, 8, better) -} - -// Tests that given a starting canonical chain of a given size, creating equal -// forks do take canonical ownership. -func TestEqualForkHeaders(t *testing.T) { - length := 10 - - // Make first chain starting from genesis - _, processor, err := newCanonical(length) - if err != nil { - t.Fatalf("failed to make new canonical chain: %v", err) - } - // Define the difficulty comparator - equal := func(td1, td2 *big.Int) { - if td2.Cmp(td1) != 0 { - t.Errorf("total difficulty mismatch: have %v, want %v", td2, td1) - } - } - // Sum of numbers must be equal to `length` for this to be an equal fork - testFork(t, processor, 0, 10, equal) - testFork(t, processor, 1, 9, equal) - testFork(t, processor, 2, 8, equal) - testFork(t, processor, 5, 5, equal) - testFork(t, processor, 6, 4, equal) - testFork(t, processor, 9, 1, equal) -} - -// Tests that chains missing links do not get accepted by the processor. -func TestBrokenHeaderChain(t *testing.T) { - // Make chain starting from genesis - db, LightChain, err := newCanonical(10) - if err != nil { - t.Fatalf("failed to make new canonical chain: %v", err) - } - // Create a forked chain, and try to insert with a missing link - chain := makeHeaderChain(LightChain.CurrentHeader(), 5, db, forkSeed)[1:] - if err := testHeaderChainImport(chain, LightChain); err == nil { - t.Errorf("broken header chain not reported") - } -} - -func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header { - var chain []*types.Header - for i, difficulty := range d { - header := &types.Header{ - Coinbase: common.Address{seed}, - Number: big.NewInt(int64(i + 1)), - Difficulty: big.NewInt(int64(difficulty)), - UncleHash: types.EmptyUncleHash, - TxHash: types.EmptyTxsHash, - ReceiptHash: types.EmptyReceiptsHash, - } - if i == 0 { - header.ParentHash = genesis.Hash() - } else { - header.ParentHash = chain[i-1].Hash() - } - chain = append(chain, types.CopyHeader(header)) - } - return chain -} - -type dummyOdr struct { - OdrBackend - db ethdb.Database - indexerConfig *IndexerConfig -} - -func (odr *dummyOdr) Database() ethdb.Database { - return odr.db -} - -func (odr *dummyOdr) Retrieve(ctx context.Context, req OdrRequest) error { - return nil -} - -func (odr *dummyOdr) IndexerConfig() *IndexerConfig { - return odr.indexerConfig -} - -// Tests that reorganizing a long difficult chain after a short easy one -// overwrites the canonical numbers and links in the database. -func TestReorgLongHeaders(t *testing.T) { - testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10) -} - -// Tests that reorganizing a short difficult chain after a long easy one -// overwrites the canonical numbers and links in the database. -func TestReorgShortHeaders(t *testing.T) { - testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11) -} - -func testReorg(t *testing.T, first, second []int, td int64) { - bc := newTestLightChain() - - // Insert an easy and a difficult chain afterwards - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, first, 11)) - bc.InsertHeaderChain(makeHeaderChainWithDiff(bc.genesisBlock, second, 22)) - // Check that the chain is valid number and link wise - prev := bc.CurrentHeader() - for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) { - if prev.ParentHash != header.Hash() { - t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash()) - } - } - // Make sure the chain total difficulty is the correct one - want := new(big.Int).Add(bc.genesisBlock.Difficulty(), big.NewInt(td)) - if have := bc.GetTd(bc.CurrentHeader().Hash(), bc.CurrentHeader().Number.Uint64()); have.Cmp(want) != 0 { - t.Errorf("total difficulty mismatch: have %v, want %v", have, want) - } -} - -// Tests that the insertion functions detect banned hashes. -func TestBadHeaderHashes(t *testing.T) { - bc := newTestLightChain() - - // Create a chain, ban a hash and try to import - var err error - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 4}, 10) - core.BadHashes[headers[2].Hash()] = true - if _, err = bc.InsertHeaderChain(headers); !errors.Is(err, core.ErrBannedHash) { - t.Errorf("error mismatch: have: %v, want %v", err, core.ErrBannedHash) - } -} - -// Tests that bad hashes are detected on boot, and the chan rolled back to a -// good state prior to the bad hash. -func TestReorgBadHeaderHashes(t *testing.T) { - bc := newTestLightChain() - - // Create a chain, import and ban afterwards - headers := makeHeaderChainWithDiff(bc.genesisBlock, []int{1, 2, 3, 4}, 10) - - if _, err := bc.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to import headers: %v", err) - } - if bc.CurrentHeader().Hash() != headers[3].Hash() { - t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash()) - } - core.BadHashes[headers[3].Hash()] = true - defer func() { delete(core.BadHashes, headers[3].Hash()) }() - - // Create a new LightChain and check that it rolled back the state. - ncm, err := NewLightChain(&dummyOdr{db: bc.chainDb}, params.TestChainConfig, ethash.NewFaker()) - if err != nil { - t.Fatalf("failed to create new chain manager: %v", err) - } - if ncm.CurrentHeader().Hash() != headers[2].Hash() { - t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash()) - } -} diff --git a/light/odr.go b/light/odr.go deleted file mode 100644 index 39f626ee2c..0000000000 --- a/light/odr.go +++ /dev/null @@ -1,196 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/trie/trienode" -) - -// NoOdr is the default context passed to an ODR capable function when the ODR -// service is not required. -var NoOdr = context.Background() - -// ErrNoPeers is returned if no peers capable of serving a queued request are available -var ErrNoPeers = errors.New("no suitable peers available") - -// OdrBackend is an interface to a backend service that handles ODR retrievals type -type OdrBackend interface { - Database() ethdb.Database - ChtIndexer() *core.ChainIndexer - BloomTrieIndexer() *core.ChainIndexer - BloomIndexer() *core.ChainIndexer - Retrieve(ctx context.Context, req OdrRequest) error - RetrieveTxStatus(ctx context.Context, req *TxStatusRequest) error - IndexerConfig() *IndexerConfig -} - -// OdrRequest is an interface for retrieval requests -type OdrRequest interface { - StoreResult(db ethdb.Database) -} - -// TrieID identifies a state or account storage trie -type TrieID struct { - BlockHash common.Hash - BlockNumber uint64 - StateRoot common.Hash - Root common.Hash - AccountAddress []byte -} - -// StateTrieID returns a TrieID for a state trie belonging to a certain block -// header. -func StateTrieID(header *types.Header) *TrieID { - return &TrieID{ - BlockHash: header.Hash(), - BlockNumber: header.Number.Uint64(), - StateRoot: header.Root, - Root: header.Root, - AccountAddress: nil, - } -} - -// StorageTrieID returns a TrieID for a contract storage trie at a given account -// of a given state trie. It also requires the root hash of the trie for -// checking Merkle proofs. -func StorageTrieID(state *TrieID, address common.Address, root common.Hash) *TrieID { - return &TrieID{ - BlockHash: state.BlockHash, - BlockNumber: state.BlockNumber, - StateRoot: state.StateRoot, - AccountAddress: address[:], - Root: root, - } -} - -// TrieRequest is the ODR request type for state/storage trie entries -type TrieRequest struct { - Id *TrieID - Key []byte - Proof *trienode.ProofSet -} - -// StoreResult stores the retrieved data in local database -func (req *TrieRequest) StoreResult(db ethdb.Database) { - req.Proof.Store(db) -} - -// CodeRequest is the ODR request type for retrieving contract code -type CodeRequest struct { - Id *TrieID // references storage trie of the account - Hash common.Hash - Data []byte -} - -// StoreResult stores the retrieved data in local database -func (req *CodeRequest) StoreResult(db ethdb.Database) { - rawdb.WriteCode(db, req.Hash, req.Data) -} - -// BlockRequest is the ODR request type for retrieving block bodies -type BlockRequest struct { - Hash common.Hash - Number uint64 - Header *types.Header - Rlp []byte -} - -// StoreResult stores the retrieved data in local database -func (req *BlockRequest) StoreResult(db ethdb.Database) { - rawdb.WriteBodyRLP(db, req.Hash, req.Number, req.Rlp) -} - -// ReceiptsRequest is the ODR request type for retrieving receipts. -type ReceiptsRequest struct { - Hash common.Hash - Number uint64 - Header *types.Header - Receipts types.Receipts -} - -// StoreResult stores the retrieved data in local database -func (req *ReceiptsRequest) StoreResult(db ethdb.Database) { - rawdb.WriteReceipts(db, req.Hash, req.Number, req.Receipts) -} - -// ChtRequest is the ODR request type for retrieving header by Canonical Hash Trie -type ChtRequest struct { - Config *IndexerConfig - ChtNum, BlockNum uint64 - ChtRoot common.Hash - Header *types.Header - Td *big.Int - Proof *trienode.ProofSet -} - -// StoreResult stores the retrieved data in local database -func (req *ChtRequest) StoreResult(db ethdb.Database) { - hash, num := req.Header.Hash(), req.Header.Number.Uint64() - rawdb.WriteHeader(db, req.Header) - rawdb.WriteTd(db, hash, num, req.Td) - rawdb.WriteCanonicalHash(db, hash, num) -} - -// BloomRequest is the ODR request type for retrieving bloom filters from a CHT structure -type BloomRequest struct { - OdrRequest - Config *IndexerConfig - BloomTrieNum uint64 - BitIdx uint - SectionIndexList []uint64 - BloomTrieRoot common.Hash - BloomBits [][]byte - Proofs *trienode.ProofSet -} - -// StoreResult stores the retrieved data in local database -func (req *BloomRequest) StoreResult(db ethdb.Database) { - for i, sectionIdx := range req.SectionIndexList { - sectionHead := rawdb.ReadCanonicalHash(db, (sectionIdx+1)*req.Config.BloomTrieSize-1) - // if we don't have the canonical hash stored for this section head number, we'll still store it under - // a key with a zero sectionHead. GetBloomBits will look there too if we still don't have the canonical - // hash. In the unlikely case we've retrieved the section head hash since then, we'll just retrieve the - // bit vector again from the network. - rawdb.WriteBloomBits(db, req.BitIdx, sectionIdx, sectionHead, req.BloomBits[i]) - } -} - -// TxStatus describes the status of a transaction -type TxStatus struct { - Status txpool.TxStatus - Lookup *rawdb.LegacyTxLookupEntry `rlp:"nil"` - Error string -} - -// TxStatusRequest is the ODR request type for retrieving transaction status -type TxStatusRequest struct { - Hashes []common.Hash - Status []TxStatus -} - -// StoreResult stores the retrieved data in local database -func (req *TxStatusRequest) StoreResult(db ethdb.Database) {} diff --git a/light/odr_test.go b/light/odr_test.go deleted file mode 100644 index de12f9b7ef..0000000000 --- a/light/odr_test.go +++ /dev/null @@ -1,339 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "bytes" - "context" - "errors" - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/trienode" -) - -var ( - testBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) - testBankFunds = big.NewInt(1_000_000_000_000_000_000) - - acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") - acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") - acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) - acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) - - testContractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") - testContractAddr common.Address -) - -type testOdr struct { - OdrBackend - indexerConfig *IndexerConfig - sdb, ldb ethdb.Database - serverState state.Database - disable bool -} - -func (odr *testOdr) Database() ethdb.Database { - return odr.ldb -} - -var ErrOdrDisabled = errors.New("ODR disabled") - -func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { - if odr.disable { - return ErrOdrDisabled - } - switch req := req.(type) { - case *BlockRequest: - number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) - if number != nil { - req.Rlp = rawdb.ReadBodyRLP(odr.sdb, req.Hash, *number) - } - case *ReceiptsRequest: - number := rawdb.ReadHeaderNumber(odr.sdb, req.Hash) - if number != nil { - req.Receipts = rawdb.ReadRawReceipts(odr.sdb, req.Hash, *number) - } - case *TrieRequest: - var ( - err error - t state.Trie - ) - if len(req.Id.AccountAddress) > 0 { - t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToAddress(req.Id.AccountAddress), req.Id.Root, nil) - } else { - t, err = odr.serverState.OpenTrie(req.Id.Root) - } - if err != nil { - panic(err) - } - nodes := trienode.NewProofSet() - t.Prove(req.Key, nodes) - req.Proof = nodes - case *CodeRequest: - req.Data = rawdb.ReadCode(odr.sdb, req.Hash) - } - req.StoreResult(odr.ldb) - return nil -} - -func (odr *testOdr) IndexerConfig() *IndexerConfig { - return odr.indexerConfig -} - -type odrTestFn func(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) - -func TestOdrGetBlockLes2(t *testing.T) { testChainOdr(t, 1, odrGetBlock) } - -func odrGetBlock(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { - var block *types.Block - if bc != nil { - block = bc.GetBlockByHash(bhash) - } else { - block, _ = lc.GetBlockByHash(ctx, bhash) - } - if block == nil { - return nil, nil - } - rlp, _ := rlp.EncodeToBytes(block) - return rlp, nil -} - -func TestOdrGetReceiptsLes2(t *testing.T) { testChainOdr(t, 1, odrGetReceipts) } - -func odrGetReceipts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { - var receipts types.Receipts - if bc != nil { - if number := rawdb.ReadHeaderNumber(db, bhash); number != nil { - if header := rawdb.ReadHeader(db, bhash, *number); header != nil { - receipts = rawdb.ReadReceipts(db, bhash, *number, header.Time, bc.Config()) - } - } - } else { - number := rawdb.ReadHeaderNumber(db, bhash) - if number != nil { - receipts, _ = GetBlockReceipts(ctx, lc.Odr(), bhash, *number) - } - } - if receipts == nil { - return nil, nil - } - rlp, _ := rlp.EncodeToBytes(receipts) - return rlp, nil -} - -func TestOdrAccountsLes2(t *testing.T) { testChainOdr(t, 1, odrAccounts) } - -func odrAccounts(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { - dummyAddr := common.HexToAddress("1234567812345678123456781234567812345678") - acc := []common.Address{testBankAddress, acc1Addr, acc2Addr, dummyAddr} - - var st *state.StateDB - if bc == nil { - header := lc.GetHeaderByHash(bhash) - st = NewState(ctx, header, lc.Odr()) - } else { - header := bc.GetHeaderByHash(bhash) - st, _ = state.New(header.Root, bc.StateCache(), nil) - } - - var res []byte - for _, addr := range acc { - bal := st.GetBalance(addr) - rlp, _ := rlp.EncodeToBytes(bal) - res = append(res, rlp...) - } - return res, st.Error() -} - -func TestOdrContractCallLes2(t *testing.T) { testChainOdr(t, 1, odrContractCall) } - -func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain, lc *LightChain, bhash common.Hash) ([]byte, error) { - data := common.Hex2Bytes("60CD26850000000000000000000000000000000000000000000000000000000000000000") - config := params.TestChainConfig - - var res []byte - for i := 0; i < 3; i++ { - data[35] = byte(i) - - var ( - st *state.StateDB - header *types.Header - chain core.ChainContext - ) - if bc == nil { - chain = lc - header = lc.GetHeaderByHash(bhash) - st = NewState(ctx, header, lc.Odr()) - } else { - chain = bc - header = bc.GetHeaderByHash(bhash) - st, _ = state.New(header.Root, bc.StateCache(), nil) - } - - // Perform read-only call. - st.SetBalance(testBankAddress, math.MaxBig256) - msg := &core.Message{ - From: testBankAddress, - To: &testContractAddr, - Value: new(big.Int), - GasLimit: 1000000, - GasPrice: big.NewInt(params.InitialBaseFee), - GasFeeCap: big.NewInt(params.InitialBaseFee), - GasTipCap: new(big.Int), - Data: data, - SkipAccountChecks: true, - } - txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(header, chain, nil) - vmenv := vm.NewEVM(context, txContext, st, config, vm.Config{NoBaseFee: true}) - gp := new(core.GasPool).AddGas(math.MaxUint64) - result, _ := core.ApplyMessage(vmenv, msg, gp) - res = append(res, result.Return()...) - if st.Error() != nil { - return res, st.Error() - } - } - return res, nil -} - -func testChainGen(i int, block *core.BlockGen) { - signer := types.HomesteadSigner{} - switch i { - case 0: - // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(10_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) - block.AddTx(tx) - case 1: - // In block 2, the test bank sends some more ether to account #1. - // acc1Addr passes it on to account #2. - // acc1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), acc1Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, testBankKey) - nonce := block.TxNonce(acc1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, acc2Addr, big.NewInt(1_000_000_000_000_000), params.TxGas, block.BaseFee(), nil), signer, acc1Key) - nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, block.BaseFee(), testContractCode), signer, acc1Key) - testContractAddr = crypto.CreateAddress(acc1Addr, nonce) - block.AddTx(tx1) - block.AddTx(tx2) - block.AddTx(tx3) - case 2: - // Block 3 is empty but was mined by account #2. - block.SetCoinbase(acc2Addr) - block.SetExtra([]byte("yeehaw")) - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000001") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) - block.AddTx(tx) - case 3: - // Block 4 includes blocks 2 and 3 as uncle headers (with modified extra data). - b2 := block.PrevBlock(1).Header() - b2.Extra = []byte("foo") - block.AddUncle(b2) - b3 := block.PrevBlock(2).Header() - b3.Extra = []byte("foo") - block.AddUncle(b3) - data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000002") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(testBankAddress), testContractAddr, big.NewInt(0), 100000, block.BaseFee(), data), signer, testBankKey) - block.AddTx(tx) - } -} - -func testChainOdr(t *testing.T, protocol int, fn odrTestFn) { - var ( - sdb = rawdb.NewMemoryDatabase() - ldb = rawdb.NewMemoryDatabase() - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - ) - // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) - _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, testChainGen) - if _, err := blockchain.InsertChain(gchain); err != nil { - t.Fatal(err) - } - - gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults)) - odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} - lightchain, err := NewLightChain(odr, gspec.Config, ethash.NewFullFaker()) - if err != nil { - t.Fatal(err) - } - headers := make([]*types.Header, len(gchain)) - for i, block := range gchain { - headers[i] = block.Header() - } - if _, err := lightchain.InsertHeaderChain(headers); err != nil { - t.Fatal(err) - } - - test := func(expFail int) { - for i := uint64(0); i <= blockchain.CurrentHeader().Number.Uint64(); i++ { - bhash := rawdb.ReadCanonicalHash(sdb, i) - b1, err := fn(NoOdr, sdb, blockchain, nil, bhash) - if err != nil { - t.Fatalf("error in full-node test for block %d: %v", i, err) - } - - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) - defer cancel() - - exp := i < uint64(expFail) - b2, err := fn(ctx, ldb, nil, lightchain, bhash) - if err != nil && exp { - t.Errorf("error in ODR test for block %d: %v", i, err) - } - - eq := bytes.Equal(b1, b2) - if exp && !eq { - t.Errorf("ODR test output for block %d doesn't match full node", i) - } - } - } - - // expect retrievals to fail (except genesis block) without a les peer - t.Log("checking without ODR") - odr.disable = true - test(1) - - // expect all retrievals to pass with ODR enabled - t.Log("checking with ODR") - odr.disable = false - test(len(gchain)) - - // still expect all retrievals to pass, now data should be cached locally - t.Log("checking without ODR, should be cached") - odr.disable = true - test(len(gchain)) -} diff --git a/light/odr_util.go b/light/odr_util.go deleted file mode 100644 index 9cac7df4fa..0000000000 --- a/light/odr_util.go +++ /dev/null @@ -1,275 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/misc/eip4844" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/rlp" -) - -// errNonCanonicalHash is returned if the requested chain data doesn't belong -// to the canonical chain. ODR can only retrieve the canonical chain data covered -// by the CHT or Bloom trie for verification. -var errNonCanonicalHash = errors.New("hash is not currently canonical") - -// GetHeaderByNumber retrieves the canonical block header corresponding to the -// given number. The returned header is proven by local CHT. -func GetHeaderByNumber(ctx context.Context, odr OdrBackend, number uint64) (*types.Header, error) { - // Try to find it in the local database first. - db := odr.Database() - hash := rawdb.ReadCanonicalHash(db, number) - - // If there is a canonical hash, there should have a header too. - // But if it's pruned, re-fetch from network again. - if (hash != common.Hash{}) { - if header := rawdb.ReadHeader(db, hash, number); header != nil { - return header, nil - } - } - // Retrieve the header via ODR, ensure the requested header is covered - // by local trusted CHT. - chts, _, chtHead := odr.ChtIndexer().Sections() - if number >= chts*odr.IndexerConfig().ChtSize { - return nil, errNoTrustedCht - } - r := &ChtRequest{ - ChtRoot: GetChtRoot(db, chts-1, chtHead), - ChtNum: chts - 1, - BlockNum: number, - Config: odr.IndexerConfig(), - } - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } - return r.Header, nil -} - -// GetCanonicalHash retrieves the canonical block hash corresponding to the number. -func GetCanonicalHash(ctx context.Context, odr OdrBackend, number uint64) (common.Hash, error) { - hash := rawdb.ReadCanonicalHash(odr.Database(), number) - if hash != (common.Hash{}) { - return hash, nil - } - header, err := GetHeaderByNumber(ctx, odr, number) - if err != nil { - return common.Hash{}, err - } - // number -> canonical mapping already be stored in db, get it. - return header.Hash(), nil -} - -// GetTd retrieves the total difficulty corresponding to the number and hash. -func GetTd(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*big.Int, error) { - td := rawdb.ReadTd(odr.Database(), hash, number) - if td != nil { - return td, nil - } - header, err := GetHeaderByNumber(ctx, odr, number) - if err != nil { - return nil, err - } - if header.Hash() != hash { - return nil, errNonCanonicalHash - } - // -> td mapping already be stored in db, get it. - return rawdb.ReadTd(odr.Database(), hash, number), nil -} - -// GetBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. -func GetBodyRLP(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (rlp.RawValue, error) { - if data := rawdb.ReadBodyRLP(odr.Database(), hash, number); data != nil { - return data, nil - } - // Retrieve the block header first and pass it for verification. - header, err := GetHeaderByNumber(ctx, odr, number) - if err != nil { - return nil, errNoHeader - } - if header.Hash() != hash { - return nil, errNonCanonicalHash - } - r := &BlockRequest{Hash: hash, Number: number, Header: header} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } - return r.Rlp, nil -} - -// GetBody retrieves the block body (transactions, uncles) corresponding to the -// hash. -func GetBody(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Body, error) { - data, err := GetBodyRLP(ctx, odr, hash, number) - if err != nil { - return nil, err - } - body := new(types.Body) - if err := rlp.DecodeBytes(data, body); err != nil { - return nil, err - } - return body, nil -} - -// GetBlock retrieves an entire block corresponding to the hash, assembling it -// back from the stored header and body. -func GetBlock(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (*types.Block, error) { - // Retrieve the block header and body contents - header, err := GetHeaderByNumber(ctx, odr, number) - if err != nil { - return nil, errNoHeader - } - body, err := GetBody(ctx, odr, hash, number) - if err != nil { - return nil, err - } - // Reassemble the block and return - return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles), nil -} - -// GetBlockReceipts retrieves the receipts generated by the transactions included -// in a block given by its hash. Receipts will be filled in with context data. -func GetBlockReceipts(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) (types.Receipts, error) { - // Assume receipts are already stored locally and attempt to retrieve. - receipts := rawdb.ReadRawReceipts(odr.Database(), hash, number) - if receipts == nil { - header, err := GetHeaderByNumber(ctx, odr, number) - if err != nil { - return nil, errNoHeader - } - if header.Hash() != hash { - return nil, errNonCanonicalHash - } - r := &ReceiptsRequest{Hash: hash, Number: number, Header: header} - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } - receipts = r.Receipts - } - // If the receipts are incomplete, fill the derived fields - if len(receipts) > 0 && receipts[0].TxHash == (common.Hash{}) { - block, err := GetBlock(ctx, odr, hash, number) - if err != nil { - return nil, err - } - genesis := rawdb.ReadCanonicalHash(odr.Database(), 0) - config := rawdb.ReadChainConfig(odr.Database(), genesis) - - var blobGasPrice *big.Int - excessBlobGas := block.ExcessBlobGas() - if excessBlobGas != nil { - blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas) - } - - if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, block.Transactions()); err != nil { - return nil, err - } - rawdb.WriteReceipts(odr.Database(), hash, number, receipts) - } - return receipts, nil -} - -// GetBlockLogs retrieves the logs generated by the transactions included in a -// block given by its hash. Logs will be filled in with context data. -func GetBlockLogs(ctx context.Context, odr OdrBackend, hash common.Hash, number uint64) ([][]*types.Log, error) { - receipts, err := GetBlockReceipts(ctx, odr, hash, number) - if err != nil { - return nil, err - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } - return logs, nil -} - -// GetBloomBits retrieves a batch of compressed bloomBits vectors belonging to -// the given bit index and section indexes. -func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint64) ([][]byte, error) { - var ( - reqIndex []int - reqSections []uint64 - db = odr.Database() - result = make([][]byte, len(sections)) - ) - blooms, _, sectionHead := odr.BloomTrieIndexer().Sections() - for i, section := range sections { - sectionHead := rawdb.ReadCanonicalHash(db, (section+1)*odr.IndexerConfig().BloomSize-1) - // If we don't have the canonical hash stored for this section head number, - // we'll still look for an entry with a zero sectionHead (we store it with - // zero section head too if we don't know it at the time of the retrieval) - if bloomBits, _ := rawdb.ReadBloomBits(db, bit, section, sectionHead); len(bloomBits) != 0 { - result[i] = bloomBits - continue - } - // TODO(rjl493456442) Convert sectionIndex to BloomTrie relative index - if section >= blooms { - return nil, errNoTrustedBloomTrie - } - reqSections = append(reqSections, section) - reqIndex = append(reqIndex, i) - } - // Find all bloombits in database, nothing to query via odr, return. - if reqSections == nil { - return result, nil - } - // Send odr request to retrieve missing bloombits. - r := &BloomRequest{ - BloomTrieRoot: GetBloomTrieRoot(db, blooms-1, sectionHead), - BloomTrieNum: blooms - 1, - BitIdx: bit, - SectionIndexList: reqSections, - Config: odr.IndexerConfig(), - } - if err := odr.Retrieve(ctx, r); err != nil { - return nil, err - } - for i, idx := range reqIndex { - result[idx] = r.BloomBits[i] - } - return result, nil -} - -// GetTransaction retrieves a canonical transaction by hash and also returns -// its position in the chain. There is no guarantee in the LES protocol that -// the mined transaction will be retrieved back for sure because of different -// reasons(the transaction is unindexed, the malicious server doesn't reply it -// deliberately, etc). Therefore, unretrieved transactions will receive a certain -// number of retries, thus giving a weak guarantee. -func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - r := &TxStatusRequest{Hashes: []common.Hash{txHash}} - if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != txpool.TxStatusIncluded { - return nil, common.Hash{}, 0, 0, err - } - pos := r.Status[0].Lookup - // first ensure that we have the header, otherwise block body retrieval will fail - // also verify if this is a canonical block by getting the header by number and checking its hash - if header, err := GetHeaderByNumber(ctx, odr, pos.BlockIndex); err != nil || header.Hash() != pos.BlockHash { - return nil, common.Hash{}, 0, 0, err - } - body, err := GetBody(ctx, odr, pos.BlockHash, pos.BlockIndex) - if err != nil || uint64(len(body.Transactions)) <= pos.Index || body.Transactions[pos.Index].Hash() != txHash { - return nil, common.Hash{}, 0, 0, err - } - return body.Transactions[pos.Index], pos.BlockHash, pos.BlockIndex, pos.Index, nil -} diff --git a/light/postprocess.go b/light/postprocess.go deleted file mode 100644 index a317e30b90..0000000000 --- a/light/postprocess.go +++ /dev/null @@ -1,538 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "bytes" - "context" - "encoding/binary" - "errors" - "fmt" - "math/big" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/trienode" -) - -// IndexerConfig includes a set of configs for chain indexers. -type IndexerConfig struct { - // The block frequency for creating CHTs. - ChtSize uint64 - - // The number of confirmations needed to generate/accept a canonical hash help trie. - ChtConfirms uint64 - - // The block frequency for creating new bloom bits. - BloomSize uint64 - - // The number of confirmation needed before a bloom section is considered probably final and its rotated bits - // are calculated. - BloomConfirms uint64 - - // The block frequency for creating BloomTrie. - BloomTrieSize uint64 - - // The number of confirmations needed to generate/accept a bloom trie. - BloomTrieConfirms uint64 -} - -var ( - // DefaultServerIndexerConfig wraps a set of configs as a default indexer config for server side. - DefaultServerIndexerConfig = &IndexerConfig{ - ChtSize: params.CHTFrequency, - ChtConfirms: params.HelperTrieProcessConfirmations, - BloomSize: params.BloomBitsBlocks, - BloomConfirms: params.BloomConfirms, - BloomTrieSize: params.BloomTrieFrequency, - BloomTrieConfirms: params.HelperTrieProcessConfirmations, - } - // DefaultClientIndexerConfig wraps a set of configs as a default indexer config for client side. - DefaultClientIndexerConfig = &IndexerConfig{ - ChtSize: params.CHTFrequency, - ChtConfirms: params.HelperTrieConfirmations, - BloomSize: params.BloomBitsBlocksClient, - BloomConfirms: params.HelperTrieConfirmations, - BloomTrieSize: params.BloomTrieFrequency, - BloomTrieConfirms: params.HelperTrieConfirmations, - } - // TestServerIndexerConfig wraps a set of configs as a test indexer config for server side. - TestServerIndexerConfig = &IndexerConfig{ - ChtSize: 128, - ChtConfirms: 1, - BloomSize: 16, - BloomConfirms: 1, - BloomTrieSize: 128, - BloomTrieConfirms: 1, - } - // TestClientIndexerConfig wraps a set of configs as a test indexer config for client side. - TestClientIndexerConfig = &IndexerConfig{ - ChtSize: 128, - ChtConfirms: 8, - BloomSize: 128, - BloomConfirms: 8, - BloomTrieSize: 128, - BloomTrieConfirms: 8, - } -) - -var ( - errNoTrustedCht = errors.New("no trusted canonical hash trie") - errNoTrustedBloomTrie = errors.New("no trusted bloom trie") - errNoHeader = errors.New("header not found") -) - -// ChtNode structures are stored in the Canonical Hash Trie in an RLP encoded format -type ChtNode struct { - Hash common.Hash - Td *big.Int -} - -// GetChtRoot reads the CHT root associated to the given section from the database -func GetChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], sectionIdx) - data, _ := db.Get(append(append(rawdb.ChtPrefix, encNumber[:]...), sectionHead.Bytes()...)) - return common.BytesToHash(data) -} - -// StoreChtRoot writes the CHT root associated to the given section into the database -func StoreChtRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], sectionIdx) - db.Put(append(append(rawdb.ChtPrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) -} - -// ChtIndexerBackend implements core.ChainIndexerBackend. -type ChtIndexerBackend struct { - disablePruning bool - diskdb, trieTable ethdb.Database - odr OdrBackend - triedb *trie.Database - section, sectionSize uint64 - lastHash common.Hash - trie *trie.Trie - originRoot common.Hash -} - -// NewChtIndexer creates a Cht chain indexer -func NewChtIndexer(db ethdb.Database, odr OdrBackend, size, confirms uint64, disablePruning bool) *core.ChainIndexer { - trieTable := rawdb.NewTable(db, string(rawdb.ChtTablePrefix)) - backend := &ChtIndexerBackend{ - diskdb: db, - odr: odr, - trieTable: trieTable, - triedb: trie.NewDatabase(trieTable, trie.HashDefaults), - sectionSize: size, - disablePruning: disablePruning, - } - return core.NewChainIndexer(db, rawdb.NewTable(db, string(rawdb.ChtIndexTablePrefix)), backend, size, confirms, time.Millisecond*100, "cht") -} - -// fetchMissingNodes tries to retrieve the last entry of the latest trusted CHT from the -// ODR backend in order to be able to add new entries and calculate subsequent root hashes -func (c *ChtIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { - batch := c.trieTable.NewBatch() - r := &ChtRequest{ChtRoot: root, ChtNum: section - 1, BlockNum: section*c.sectionSize - 1, Config: c.odr.IndexerConfig()} - for { - err := c.odr.Retrieve(ctx, r) - switch err { - case nil: - r.Proof.Store(batch) - return batch.Write() - case ErrNoPeers: - // if there are no peers to serve, retry later - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Second * 10): - // stay in the loop and try again - } - default: - return err - } - } -} - -// Reset implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { - root := types.EmptyRootHash - if section > 0 { - root = GetChtRoot(c.diskdb, section-1, lastSectionHead) - } - var err error - c.trie, err = trie.New(trie.TrieID(root), c.triedb) - - if err != nil && c.odr != nil { - err = c.fetchMissingNodes(ctx, section, root) - if err == nil { - c.trie, err = trie.New(trie.TrieID(root), c.triedb) - } - } - c.section = section - c.originRoot = root - return err -} - -// Process implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) error { - hash, num := header.Hash(), header.Number.Uint64() - c.lastHash = hash - - td := rawdb.ReadTd(c.diskdb, hash, num) - if td == nil { - panic(nil) - } - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], num) - data, _ := rlp.EncodeToBytes(ChtNode{hash, td}) - return c.trie.Update(encNumber[:], data) -} - -// Commit implements core.ChainIndexerBackend -func (c *ChtIndexerBackend) Commit() error { - root, nodes, err := c.trie.Commit(false) - if err != nil { - return err - } - // Commit trie changes into trie database in case it's not nil. - if nodes != nil { - if err := c.triedb.Update(root, c.originRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { - return err - } - if err := c.triedb.Commit(root, false); err != nil { - return err - } - } - // Re-create trie with newly generated root and updated database. - c.trie, err = trie.New(trie.TrieID(root), c.triedb) - if err != nil { - return err - } - // Pruning historical trie nodes if necessary. - if !c.disablePruning { - it := c.trieTable.NewIterator(nil, nil) - defer it.Release() - - var ( - deleted int - batch = c.trieTable.NewBatch() - t = time.Now() - ) - hashes := make(map[common.Hash]struct{}) - if nodes != nil { - for _, hash := range nodes.Hashes() { - hashes[hash] = struct{}{} - } - } - for it.Next() { - trimmed := bytes.TrimPrefix(it.Key(), rawdb.ChtTablePrefix) - if len(trimmed) == common.HashLength { - if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { - batch.Delete(trimmed) - deleted += 1 - } - } - } - if err := batch.Write(); err != nil { - return err - } - log.Debug("Prune historical CHT trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) - } - log.Info("Storing CHT", "section", c.section, "head", fmt.Sprintf("%064x", c.lastHash), "root", fmt.Sprintf("%064x", root)) - StoreChtRoot(c.diskdb, c.section, c.lastHash, root) - return nil -} - -// Prune implements core.ChainIndexerBackend which deletes all chain data -// (except hash<->number mappings) older than the specified threshold. -func (c *ChtIndexerBackend) Prune(threshold uint64) error { - // Short circuit if the light pruning is disabled. - if c.disablePruning { - return nil - } - t := time.Now() - // Always keep genesis header in database. - start, end := uint64(1), (threshold+1)*c.sectionSize - - var batch = c.diskdb.NewBatch() - for { - numbers, hashes := rawdb.ReadAllCanonicalHashes(c.diskdb, start, end, 10240) - if len(numbers) == 0 { - break - } - for i := 0; i < len(numbers); i++ { - // Keep hash<->number mapping in database otherwise the hash based - // API(e.g. GetReceipt, GetLogs) will be broken. - // - // Storage size wise, the size of a mapping is ~41bytes. For one - // section is about 1.3MB which is acceptable. - // - // In order to totally get rid of this index, we need an additional - // flag to specify how many historical data light client can serve. - rawdb.DeleteCanonicalHash(batch, numbers[i]) - rawdb.DeleteBlockWithoutNumber(batch, hashes[i], numbers[i]) - } - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } - start = numbers[len(numbers)-1] + 1 - } - if err := batch.Write(); err != nil { - return err - } - log.Debug("Prune history headers", "threshold", threshold, "elapsed", common.PrettyDuration(time.Since(t))) - return nil -} - -// GetBloomTrieRoot reads the BloomTrie root associated to the given section from the database -func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], sectionIdx) - data, _ := db.Get(append(append(rawdb.BloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...)) - return common.BytesToHash(data) -} - -// StoreBloomTrieRoot writes the BloomTrie root associated to the given section into the database -func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { - var encNumber [8]byte - binary.BigEndian.PutUint64(encNumber[:], sectionIdx) - db.Put(append(append(rawdb.BloomTriePrefix, encNumber[:]...), sectionHead.Bytes()...), root.Bytes()) -} - -// BloomTrieIndexerBackend implements core.ChainIndexerBackend -type BloomTrieIndexerBackend struct { - disablePruning bool - diskdb, trieTable ethdb.Database - triedb *trie.Database - odr OdrBackend - section uint64 - parentSize uint64 - size uint64 - bloomTrieRatio uint64 - trie *trie.Trie - originRoot common.Hash - sectionHeads []common.Hash -} - -// NewBloomTrieIndexer creates a BloomTrie chain indexer -func NewBloomTrieIndexer(db ethdb.Database, odr OdrBackend, parentSize, size uint64, disablePruning bool) *core.ChainIndexer { - trieTable := rawdb.NewTable(db, string(rawdb.BloomTrieTablePrefix)) - backend := &BloomTrieIndexerBackend{ - diskdb: db, - odr: odr, - trieTable: trieTable, - triedb: trie.NewDatabase(trieTable, trie.HashDefaults), - parentSize: parentSize, - size: size, - disablePruning: disablePruning, - } - backend.bloomTrieRatio = size / parentSize - backend.sectionHeads = make([]common.Hash, backend.bloomTrieRatio) - return core.NewChainIndexer(db, rawdb.NewTable(db, string(rawdb.BloomTrieIndexPrefix)), backend, size, 0, time.Millisecond*100, "bloomtrie") -} - -// fetchMissingNodes tries to retrieve the last entries of the latest trusted bloom trie from the -// ODR backend in order to be able to add new entries and calculate subsequent root hashes -func (b *BloomTrieIndexerBackend) fetchMissingNodes(ctx context.Context, section uint64, root common.Hash) error { - indexCh := make(chan uint, types.BloomBitLength) - type res struct { - nodes *trienode.ProofSet - err error - } - resCh := make(chan res, types.BloomBitLength) - for i := 0; i < 20; i++ { - go func() { - for bitIndex := range indexCh { - r := &BloomRequest{BloomTrieRoot: root, BloomTrieNum: section - 1, BitIdx: bitIndex, SectionIndexList: []uint64{section - 1}, Config: b.odr.IndexerConfig()} - for { - if err := b.odr.Retrieve(ctx, r); err == ErrNoPeers { - // if there are no peers to serve, retry later - select { - case <-ctx.Done(): - resCh <- res{nil, ctx.Err()} - return - case <-time.After(time.Second * 10): - // stay in the loop and try again - } - } else { - resCh <- res{r.Proofs, err} - break - } - } - } - }() - } - for i := uint(0); i < types.BloomBitLength; i++ { - indexCh <- i - } - close(indexCh) - batch := b.trieTable.NewBatch() - for i := uint(0); i < types.BloomBitLength; i++ { - res := <-resCh - if res.err != nil { - return res.err - } - res.nodes.Store(batch) - } - return batch.Write() -} - -// Reset implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { - root := types.EmptyRootHash - if section > 0 { - root = GetBloomTrieRoot(b.diskdb, section-1, lastSectionHead) - } - var err error - b.trie, err = trie.New(trie.TrieID(root), b.triedb) - if err != nil && b.odr != nil { - err = b.fetchMissingNodes(ctx, section, root) - if err == nil { - b.trie, err = trie.New(trie.TrieID(root), b.triedb) - } - } - b.section = section - b.originRoot = root - return err -} - -// Process implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Process(ctx context.Context, header *types.Header) error { - num := header.Number.Uint64() - b.section*b.size - if (num+1)%b.parentSize == 0 { - b.sectionHeads[num/b.parentSize] = header.Hash() - } - return nil -} - -// Commit implements core.ChainIndexerBackend -func (b *BloomTrieIndexerBackend) Commit() error { - var compSize, decompSize uint64 - - for i := uint(0); i < types.BloomBitLength; i++ { - var encKey [10]byte - binary.BigEndian.PutUint16(encKey[0:2], uint16(i)) - binary.BigEndian.PutUint64(encKey[2:10], b.section) - var decomp []byte - for j := uint64(0); j < b.bloomTrieRatio; j++ { - data, err := rawdb.ReadBloomBits(b.diskdb, i, b.section*b.bloomTrieRatio+j, b.sectionHeads[j]) - if err != nil { - return err - } - decompData, err2 := bitutil.DecompressBytes(data, int(b.parentSize/8)) - if err2 != nil { - return err2 - } - decomp = append(decomp, decompData...) - } - comp := bitutil.CompressBytes(decomp) - - decompSize += uint64(len(decomp)) - compSize += uint64(len(comp)) - - var terr error - if len(comp) > 0 { - terr = b.trie.Update(encKey[:], comp) - } else { - terr = b.trie.Delete(encKey[:]) - } - if terr != nil { - return terr - } - } - root, nodes, err := b.trie.Commit(false) - if err != nil { - return err - } - // Commit trie changes into trie database in case it's not nil. - if nodes != nil { - if err := b.triedb.Update(root, b.originRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { - return err - } - if err := b.triedb.Commit(root, false); err != nil { - return err - } - } - // Re-create trie with newly generated root and updated database. - b.trie, err = trie.New(trie.TrieID(root), b.triedb) - if err != nil { - return err - } - // Pruning historical trie nodes if necessary. - if !b.disablePruning { - it := b.trieTable.NewIterator(nil, nil) - defer it.Release() - - var ( - deleted int - batch = b.trieTable.NewBatch() - t = time.Now() - ) - hashes := make(map[common.Hash]struct{}) - if nodes != nil { - for _, hash := range nodes.Hashes() { - hashes[hash] = struct{}{} - } - } - for it.Next() { - trimmed := bytes.TrimPrefix(it.Key(), rawdb.BloomTrieTablePrefix) - if len(trimmed) == common.HashLength { - if _, ok := hashes[common.BytesToHash(trimmed)]; !ok { - batch.Delete(trimmed) - deleted += 1 - } - } - } - if err := batch.Write(); err != nil { - return err - } - log.Debug("Prune historical bloom trie nodes", "deleted", deleted, "remaining", len(hashes), "elapsed", common.PrettyDuration(time.Since(t))) - } - sectionHead := b.sectionHeads[b.bloomTrieRatio-1] - StoreBloomTrieRoot(b.diskdb, b.section, sectionHead, root) - log.Info("Storing bloom trie", "section", b.section, "head", fmt.Sprintf("%064x", sectionHead), "root", fmt.Sprintf("%064x", root), "compression", float64(compSize)/float64(decompSize)) - - return nil -} - -// Prune implements core.ChainIndexerBackend which deletes all -// bloombits which older than the specified threshold. -func (b *BloomTrieIndexerBackend) Prune(threshold uint64) error { - // Short circuit if the light pruning is disabled. - if b.disablePruning { - return nil - } - start := time.Now() - for i := uint(0); i < types.BloomBitLength; i++ { - rawdb.DeleteBloombits(b.diskdb, i, 0, threshold*b.bloomTrieRatio+b.bloomTrieRatio) - } - log.Debug("Prune history bloombits", "threshold", threshold, "elapsed", common.PrettyDuration(time.Since(start))) - return nil -} diff --git a/light/trie.go b/light/trie.go deleted file mode 100644 index 1d93bdf415..0000000000 --- a/light/trie.go +++ /dev/null @@ -1,319 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie" - "github.com/ethereum/go-ethereum/trie/trienode" -) - -var ( - sha3Nil = crypto.Keccak256Hash(nil) -) - -func NewState(ctx context.Context, head *types.Header, odr OdrBackend) *state.StateDB { - state, _ := state.New(head.Root, NewStateDatabase(ctx, head, odr), nil) - return state -} - -func NewStateDatabase(ctx context.Context, head *types.Header, odr OdrBackend) state.Database { - return &odrDatabase{ctx, StateTrieID(head), odr} -} - -type odrDatabase struct { - ctx context.Context - id *TrieID - backend OdrBackend -} - -func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { - return &odrTrie{db: db, id: db.id}, nil -} - -func (db *odrDatabase) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ state.Trie) (state.Trie, error) { - return &odrTrie{db: db, id: StorageTrieID(db.id, address, root)}, nil -} - -func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { - switch t := t.(type) { - case *odrTrie: - cpy := &odrTrie{db: t.db, id: t.id} - if t.trie != nil { - cpy.trie = t.trie.Copy() - } - return cpy - default: - panic(fmt.Errorf("unknown trie type %T", t)) - } -} - -func (db *odrDatabase) ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) { - if codeHash == sha3Nil { - return nil, nil - } - code := rawdb.ReadCode(db.backend.Database(), codeHash) - if len(code) != 0 { - return code, nil - } - id := *db.id - id.AccountAddress = addr[:] - req := &CodeRequest{Id: &id, Hash: codeHash} - err := db.backend.Retrieve(db.ctx, req) - return req.Data, err -} - -func (db *odrDatabase) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { - code, err := db.ContractCode(addr, codeHash) - return len(code), err -} - -func (db *odrDatabase) TrieDB() *trie.Database { - return nil -} - -func (db *odrDatabase) DiskDB() ethdb.KeyValueStore { - panic("not implemented") -} - -type odrTrie struct { - db *odrDatabase - id *TrieID - trie *trie.Trie -} - -func (t *odrTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { - key = crypto.Keccak256(key) - var enc []byte - err := t.do(key, func() (err error) { - enc, err = t.trie.Get(key) - return err - }) - if err != nil || len(enc) == 0 { - return nil, err - } - _, content, _, err := rlp.Split(enc) - return content, err -} - -func (t *odrTrie) GetAccount(address common.Address) (*types.StateAccount, error) { - var ( - enc []byte - key = crypto.Keccak256(address.Bytes()) - ) - err := t.do(key, func() (err error) { - enc, err = t.trie.Get(key) - return err - }) - if err != nil || len(enc) == 0 { - return nil, err - } - acct := new(types.StateAccount) - if err := rlp.DecodeBytes(enc, acct); err != nil { - return nil, err - } - return acct, nil -} - -func (t *odrTrie) UpdateAccount(address common.Address, acc *types.StateAccount) error { - key := crypto.Keccak256(address.Bytes()) - value, err := rlp.EncodeToBytes(acc) - if err != nil { - return fmt.Errorf("decoding error in account update: %w", err) - } - return t.do(key, func() error { - return t.trie.Update(key, value) - }) -} - -func (t *odrTrie) UpdateContractCode(_ common.Address, _ common.Hash, _ []byte) error { - return nil -} - -func (t *odrTrie) UpdateStorage(_ common.Address, key, value []byte) error { - key = crypto.Keccak256(key) - v, _ := rlp.EncodeToBytes(value) - return t.do(key, func() error { - return t.trie.Update(key, v) - }) -} - -func (t *odrTrie) DeleteStorage(_ common.Address, key []byte) error { - key = crypto.Keccak256(key) - return t.do(key, func() error { - return t.trie.Delete(key) - }) -} - -// DeleteAccount abstracts an account deletion from the trie. -func (t *odrTrie) DeleteAccount(address common.Address) error { - key := crypto.Keccak256(address.Bytes()) - return t.do(key, func() error { - return t.trie.Delete(key) - }) -} - -func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { - if t.trie == nil { - return t.id.Root, nil, nil - } - return t.trie.Commit(collectLeaf) -} - -func (t *odrTrie) Hash() common.Hash { - if t.trie == nil { - return t.id.Root - } - return t.trie.Hash() -} - -func (t *odrTrie) NodeIterator(startkey []byte) (trie.NodeIterator, error) { - return newNodeIterator(t, startkey), nil -} - -func (t *odrTrie) GetKey(sha []byte) []byte { - return nil -} - -func (t *odrTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { - return errors.New("not implemented, needs client/server interface split") -} - -// do tries and retries to execute a function until it returns with no error or -// an error type other than MissingNodeError -func (t *odrTrie) do(key []byte, fn func() error) error { - for { - var err error - if t.trie == nil { - var id *trie.ID - if len(t.id.AccountAddress) > 0 { - id = trie.StorageTrieID(t.id.StateRoot, crypto.Keccak256Hash(t.id.AccountAddress), t.id.Root) - } else { - id = trie.StateTrieID(t.id.StateRoot) - } - triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults) - t.trie, err = trie.New(id, triedb) - } - if err == nil { - err = fn() - } - if _, ok := err.(*trie.MissingNodeError); !ok { - return err - } - r := &TrieRequest{Id: t.id, Key: key} - if err := t.db.backend.Retrieve(t.db.ctx, r); err != nil { - return err - } - } -} - -type nodeIterator struct { - trie.NodeIterator - t *odrTrie - err error -} - -func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { - it := &nodeIterator{t: t} - // Open the actual non-ODR trie if that hasn't happened yet. - if t.trie == nil { - it.do(func() error { - var id *trie.ID - if len(t.id.AccountAddress) > 0 { - id = trie.StorageTrieID(t.id.StateRoot, crypto.Keccak256Hash(t.id.AccountAddress), t.id.Root) - } else { - id = trie.StateTrieID(t.id.StateRoot) - } - triedb := trie.NewDatabase(t.db.backend.Database(), trie.HashDefaults) - t, err := trie.New(id, triedb) - if err == nil { - it.t.trie = t - } - return err - }) - } - it.do(func() error { - var err error - it.NodeIterator, err = it.t.trie.NodeIterator(startkey) - if err != nil { - return err - } - return it.NodeIterator.Error() - }) - return it -} - -func (it *nodeIterator) Next(descend bool) bool { - var ok bool - it.do(func() error { - ok = it.NodeIterator.Next(descend) - return it.NodeIterator.Error() - }) - return ok -} - -// do runs fn and attempts to fill in missing nodes by retrieving. -func (it *nodeIterator) do(fn func() error) { - var lasthash common.Hash - for { - it.err = fn() - missing, ok := it.err.(*trie.MissingNodeError) - if !ok { - return - } - if missing.NodeHash == lasthash { - it.err = fmt.Errorf("retrieve loop for trie node %x", missing.NodeHash) - return - } - lasthash = missing.NodeHash - r := &TrieRequest{Id: it.t.id, Key: nibblesToKey(missing.Path)} - if it.err = it.t.db.backend.Retrieve(it.t.db.ctx, r); it.err != nil { - return - } - } -} - -func (it *nodeIterator) Error() error { - if it.err != nil { - return it.err - } - return it.NodeIterator.Error() -} - -func nibblesToKey(nib []byte) []byte { - if len(nib) > 0 && nib[len(nib)-1] == 0x10 { - nib = nib[:len(nib)-1] // drop terminator - } - if len(nib)&1 == 1 { - nib = append(nib, 0) // make even - } - key := make([]byte, len(nib)/2) - for bi, ni := 0, 0; ni < len(nib); bi, ni = bi+1, ni+2 { - key[bi] = nib[ni]<<4 | nib[ni+1] - } - return key -} diff --git a/light/trie_test.go b/light/trie_test.go deleted file mode 100644 index fe724e9eea..0000000000 --- a/light/trie_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "bytes" - "context" - "errors" - "fmt" - "math/big" - "testing" - - "github.com/davecgh/go-spew/spew" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -func TestNodeIterator(t *testing.T) { - var ( - fulldb = rawdb.NewMemoryDatabase() - lightdb = rawdb.NewMemoryDatabase() - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - ) - blockchain, _ := core.NewBlockChain(fulldb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) - _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), 4, testChainGen) - if _, err := blockchain.InsertChain(gchain); err != nil { - panic(err) - } - - gspec.MustCommit(lightdb, trie.NewDatabase(lightdb, trie.HashDefaults)) - ctx := context.Background() - odr := &testOdr{sdb: fulldb, ldb: lightdb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} - head := blockchain.CurrentHeader() - lightTrie, _ := NewStateDatabase(ctx, head, odr).OpenTrie(head.Root) - fullTrie, _ := blockchain.StateCache().OpenTrie(head.Root) - if err := diffTries(fullTrie, lightTrie); err != nil { - t.Fatal(err) - } -} - -func diffTries(t1, t2 state.Trie) error { - trieIt1, err := t1.NodeIterator(nil) - if err != nil { - return err - } - trieIt2, err := t2.NodeIterator(nil) - if err != nil { - return err - } - i1 := trie.NewIterator(trieIt1) - i2 := trie.NewIterator(trieIt2) - for i1.Next() && i2.Next() { - if !bytes.Equal(i1.Key, i2.Key) { - spew.Dump(i2) - return fmt.Errorf("tries have different keys %x, %x", i1.Key, i2.Key) - } - if !bytes.Equal(i1.Value, i2.Value) { - return fmt.Errorf("tries differ at key %x", i1.Key) - } - } - switch { - case i1.Err != nil: - return fmt.Errorf("full trie iterator error: %v", i1.Err) - case i2.Err != nil: - return fmt.Errorf("light trie iterator error: %v", i2.Err) - case i1.Next(): - return errors.New("full trie iterator has more k/v pairs") - case i2.Next(): - return errors.New("light trie iterator has more k/v pairs") - } - return nil -} diff --git a/light/txpool.go b/light/txpool.go deleted file mode 100644 index b792d70b14..0000000000 --- a/light/txpool.go +++ /dev/null @@ -1,556 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "fmt" - "math/big" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/txpool" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" -) - -const ( - // chainHeadChanSize is the size of channel listening to ChainHeadEvent. - chainHeadChanSize = 10 -) - -// txPermanent is the number of mined blocks after a mined transaction is -// considered permanent and no rollback is expected -var txPermanent = uint64(500) - -// TxPool implements the transaction pool for light clients, which keeps track -// of the status of locally created transactions, detecting if they are included -// in a block (mined) or rolled back. There are no queued transactions since we -// always receive all locally signed transactions in the same order as they are -// created. -type TxPool struct { - config *params.ChainConfig - signer types.Signer - quit chan bool - txFeed event.Feed - scope event.SubscriptionScope - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - mu sync.RWMutex - chain *LightChain - odr OdrBackend - chainDb ethdb.Database - relay TxRelayBackend - head common.Hash - nonce map[common.Address]uint64 // "pending" nonce - pending map[common.Hash]*types.Transaction // pending transactions by tx hash - mined map[common.Hash][]*types.Transaction // mined transactions by block hash - clearIdx uint64 // earliest block nr that can contain mined tx info - - istanbul bool // Fork indicator whether we are in the istanbul stage. - eip2718 bool // Fork indicator whether we are in the eip2718 stage. - shanghai bool // Fork indicator whether we are in the shanghai stage. -} - -// TxRelayBackend provides an interface to the mechanism that forwards transactions to the -// ETH network. The implementations of the functions should be non-blocking. -// -// Send instructs backend to forward new transactions NewHead notifies backend about a new -// head after processed by the tx pool, including mined and rolled back transactions since -// the last event. -// -// Discard notifies backend about transactions that should be discarded either because -// they have been replaced by a re-send or because they have been mined long ago and no -// rollback is expected. -type TxRelayBackend interface { - Send(txs types.Transactions) - NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) - Discard(hashes []common.Hash) -} - -// NewTxPool creates a new light transaction pool -func NewTxPool(config *params.ChainConfig, chain *LightChain, relay TxRelayBackend) *TxPool { - pool := &TxPool{ - config: config, - signer: types.LatestSigner(config), - nonce: make(map[common.Address]uint64), - pending: make(map[common.Hash]*types.Transaction), - mined: make(map[common.Hash][]*types.Transaction), - quit: make(chan bool), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - chain: chain, - relay: relay, - odr: chain.Odr(), - chainDb: chain.Odr().Database(), - head: chain.CurrentHeader().Hash(), - clearIdx: chain.CurrentHeader().Number.Uint64(), - } - // Subscribe events from blockchain - pool.chainHeadSub = pool.chain.SubscribeChainHeadEvent(pool.chainHeadCh) - go pool.eventLoop() - - return pool -} - -// currentState returns the light state of the current head header -func (pool *TxPool) currentState(ctx context.Context) *state.StateDB { - return NewState(ctx, pool.chain.CurrentHeader(), pool.odr) -} - -// GetNonce returns the "pending" nonce of a given address. It always queries -// the nonce belonging to the latest header too in order to detect if another -// client using the same key sent a transaction. -func (pool *TxPool) GetNonce(ctx context.Context, addr common.Address) (uint64, error) { - state := pool.currentState(ctx) - nonce := state.GetNonce(addr) - if state.Error() != nil { - return 0, state.Error() - } - sn, ok := pool.nonce[addr] - if ok && sn > nonce { - nonce = sn - } - if !ok || sn < nonce { - pool.nonce[addr] = nonce - } - return nonce, nil -} - -// txStateChanges stores the recent changes between pending/mined states of -// transactions. True means mined, false means rolled back, no entry means no change -type txStateChanges map[common.Hash]bool - -// setState sets the status of a tx to either recently mined or recently rolled back -func (txc txStateChanges) setState(txHash common.Hash, mined bool) { - val, ent := txc[txHash] - if ent && (val != mined) { - delete(txc, txHash) - } else { - txc[txHash] = mined - } -} - -// getLists creates lists of mined and rolled back tx hashes -func (txc txStateChanges) getLists() (mined []common.Hash, rollback []common.Hash) { - for hash, val := range txc { - if val { - mined = append(mined, hash) - } else { - rollback = append(rollback, hash) - } - } - return -} - -// checkMinedTxs checks newly added blocks for the currently pending transactions -// and marks them as mined if necessary. It also stores block position in the db -// and adds them to the received txStateChanges map. -func (pool *TxPool) checkMinedTxs(ctx context.Context, hash common.Hash, number uint64, txc txStateChanges) error { - // If no transactions are pending, we don't care about anything - if len(pool.pending) == 0 { - return nil - } - block, err := GetBlock(ctx, pool.odr, hash, number) - if err != nil { - return err - } - // Gather all the local transaction mined in this block - list := pool.mined[hash] - for _, tx := range block.Transactions() { - if _, ok := pool.pending[tx.Hash()]; ok { - list = append(list, tx) - } - } - // If some transactions have been mined, write the needed data to disk and update - if list != nil { - // Retrieve all the receipts belonging to this block and write the lookup table - if _, err := GetBlockReceipts(ctx, pool.odr, hash, number); err != nil { // ODR caches, ignore results - return err - } - rawdb.WriteTxLookupEntriesByBlock(pool.chainDb, block) - - // Update the transaction pool's state - for _, tx := range list { - delete(pool.pending, tx.Hash()) - txc.setState(tx.Hash(), true) - } - pool.mined[hash] = list - } - return nil -} - -// rollbackTxs marks the transactions contained in recently rolled back blocks -// as rolled back. It also removes any positional lookup entries. -func (pool *TxPool) rollbackTxs(hash common.Hash, txc txStateChanges) { - batch := pool.chainDb.NewBatch() - if list, ok := pool.mined[hash]; ok { - for _, tx := range list { - txHash := tx.Hash() - rawdb.DeleteTxLookupEntry(batch, txHash) - pool.pending[txHash] = tx - txc.setState(txHash, false) - } - delete(pool.mined, hash) - } - batch.Write() -} - -// reorgOnNewHead sets a new head header, processing (and rolling back if necessary) -// the blocks since the last known head and returns a txStateChanges map containing -// the recently mined and rolled back transaction hashes. If an error (context -// timeout) occurs during checking new blocks, it leaves the locally known head -// at the latest checked block and still returns a valid txStateChanges, making it -// possible to continue checking the missing blocks at the next chain head event -func (pool *TxPool) reorgOnNewHead(ctx context.Context, newHeader *types.Header) (txStateChanges, error) { - txc := make(txStateChanges) - oldh := pool.chain.GetHeaderByHash(pool.head) - newh := newHeader - // find common ancestor, create list of rolled back and new block hashes - var oldHashes, newHashes []common.Hash - for oldh.Hash() != newh.Hash() { - if oldh.Number.Uint64() >= newh.Number.Uint64() { - oldHashes = append(oldHashes, oldh.Hash()) - oldh = pool.chain.GetHeader(oldh.ParentHash, oldh.Number.Uint64()-1) - } - if oldh.Number.Uint64() < newh.Number.Uint64() { - newHashes = append(newHashes, newh.Hash()) - newh = pool.chain.GetHeader(newh.ParentHash, newh.Number.Uint64()-1) - if newh == nil { - // happens when CHT syncing, nothing to do - newh = oldh - } - } - } - if oldh.Number.Uint64() < pool.clearIdx { - pool.clearIdx = oldh.Number.Uint64() - } - // roll back old blocks - for _, hash := range oldHashes { - pool.rollbackTxs(hash, txc) - } - pool.head = oldh.Hash() - // check mined txs of new blocks (array is in reversed order) - for i := len(newHashes) - 1; i >= 0; i-- { - hash := newHashes[i] - if err := pool.checkMinedTxs(ctx, hash, newHeader.Number.Uint64()-uint64(i), txc); err != nil { - return txc, err - } - pool.head = hash - } - - // clear old mined tx entries of old blocks - if idx := newHeader.Number.Uint64(); idx > pool.clearIdx+txPermanent { - idx2 := idx - txPermanent - if len(pool.mined) > 0 { - for i := pool.clearIdx; i < idx2; i++ { - hash := rawdb.ReadCanonicalHash(pool.chainDb, i) - if list, ok := pool.mined[hash]; ok { - hashes := make([]common.Hash, len(list)) - for i, tx := range list { - hashes[i] = tx.Hash() - } - pool.relay.Discard(hashes) - delete(pool.mined, hash) - } - } - } - pool.clearIdx = idx2 - } - - return txc, nil -} - -// blockCheckTimeout is the time limit for checking new blocks for mined -// transactions. Checking resumes at the next chain head event if timed out. -const blockCheckTimeout = time.Second * 3 - -// eventLoop processes chain head events and also notifies the tx relay backend -// about the new head hash and tx state changes -func (pool *TxPool) eventLoop() { - for { - select { - case ev := <-pool.chainHeadCh: - pool.setNewHead(ev.Block.Header()) - // hack in order to avoid hogging the lock; this part will - // be replaced by a subsequent PR. - time.Sleep(time.Millisecond) - - // System stopped - case <-pool.chainHeadSub.Err(): - return - } - } -} - -func (pool *TxPool) setNewHead(head *types.Header) { - pool.mu.Lock() - defer pool.mu.Unlock() - - ctx, cancel := context.WithTimeout(context.Background(), blockCheckTimeout) - defer cancel() - - txc, _ := pool.reorgOnNewHead(ctx, head) - m, r := txc.getLists() - pool.relay.NewHead(pool.head, m, r) - - // Update fork indicator by next pending block number - next := new(big.Int).Add(head.Number, big.NewInt(1)) - pool.istanbul = pool.config.IsIstanbul(next) - pool.eip2718 = pool.config.IsBerlin(next) - pool.shanghai = pool.config.IsShanghai(next, uint64(time.Now().Unix())) -} - -// Stop stops the light transaction pool -func (pool *TxPool) Stop() { - // Unsubscribe all subscriptions registered from txpool - pool.scope.Close() - // Unsubscribe subscriptions registered from blockchain - pool.chainHeadSub.Unsubscribe() - close(pool.quit) - log.Info("Transaction pool stopped") -} - -// SubscribeNewTxsEvent registers a subscription of core.NewTxsEvent and -// starts sending event to the given channel. -func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return pool.scope.Track(pool.txFeed.Subscribe(ch)) -} - -// Stats returns the number of currently pending (locally created) transactions -func (pool *TxPool) Stats() (pending int) { - pool.mu.RLock() - defer pool.mu.RUnlock() - - pending = len(pool.pending) - return -} - -// validateTx checks whether a transaction is valid according to the consensus rules. -func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error { - // Validate sender - var ( - from common.Address - err error - ) - - // Validate the transaction sender and it's sig. Throw - // if the from fields is invalid. - if from, err = types.Sender(pool.signer, tx); err != nil { - return txpool.ErrInvalidSender - } - // Last but not least check for nonce errors - currentState := pool.currentState(ctx) - if n := currentState.GetNonce(from); n > tx.Nonce() { - return core.ErrNonceTooLow - } - - // Check the transaction doesn't exceed the current - // block limit gas. - header := pool.chain.GetHeaderByHash(pool.head) - if header.GasLimit < tx.Gas() { - return txpool.ErrGasLimit - } - - // Transactions can't be negative. This may never happen - // using RLP decoded transactions but may occur if you create - // a transaction using the RPC for example. - if tx.Value().Sign() < 0 { - return txpool.ErrNegativeValue - } - - // Transactor should have enough funds to cover the costs - // cost == V + GP * GL - if b := currentState.GetBalance(from); b.Cmp(tx.Cost()) < 0 { - return core.ErrInsufficientFunds - } - - // Should supply enough intrinsic gas - gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.To() == nil, true, pool.istanbul, pool.shanghai) - if err != nil { - return err - } - if tx.Gas() < gas { - return core.ErrIntrinsicGas - } - return currentState.Error() -} - -// add validates a new transaction and sets its state pending if processable. -// It also updates the locally stored nonce if necessary. -func (pool *TxPool) add(ctx context.Context, tx *types.Transaction) error { - hash := tx.Hash() - - if pool.pending[hash] != nil { - return fmt.Errorf("known transaction (%x)", hash[:4]) - } - err := pool.validateTx(ctx, tx) - if err != nil { - return err - } - - if _, ok := pool.pending[hash]; !ok { - pool.pending[hash] = tx - - nonce := tx.Nonce() + 1 - - addr, _ := types.Sender(pool.signer, tx) - if nonce > pool.nonce[addr] { - pool.nonce[addr] = nonce - } - - // Notify the subscribers. This event is posted in a goroutine - // because it's possible that somewhere during the post "Remove transaction" - // gets called which will then wait for the global tx pool lock and deadlock. - go pool.txFeed.Send(core.NewTxsEvent{Txs: types.Transactions{tx}}) - } - - // Print a log message if low enough level is set - log.Debug("Pooled new transaction", "hash", hash, "from", log.Lazy{Fn: func() common.Address { from, _ := types.Sender(pool.signer, tx); return from }}, "to", tx.To()) - return nil -} - -// Add adds a transaction to the pool if valid and passes it to the tx relay -// backend -func (pool *TxPool) Add(ctx context.Context, tx *types.Transaction) error { - pool.mu.Lock() - defer pool.mu.Unlock() - data, err := tx.MarshalBinary() - if err != nil { - return err - } - - if err := pool.add(ctx, tx); err != nil { - return err - } - //fmt.Println("Send", tx.Hash()) - pool.relay.Send(types.Transactions{tx}) - - pool.chainDb.Put(tx.Hash().Bytes(), data) - return nil -} - -// AddBatch adds all valid transactions to the pool and passes them to -// the tx relay backend -func (pool *TxPool) AddBatch(ctx context.Context, txs []*types.Transaction) { - pool.mu.Lock() - defer pool.mu.Unlock() - var sendTx types.Transactions - - for _, tx := range txs { - if err := pool.add(ctx, tx); err == nil { - sendTx = append(sendTx, tx) - } - } - if len(sendTx) > 0 { - pool.relay.Send(sendTx) - } -} - -// GetTransaction returns a transaction if it is contained in the pool -// and nil otherwise. -func (pool *TxPool) GetTransaction(hash common.Hash) *types.Transaction { - // check the txs first - if tx, ok := pool.pending[hash]; ok { - return tx - } - return nil -} - -// GetTransactions returns all currently processable transactions. -// The returned slice may be modified by the caller. -func (pool *TxPool) GetTransactions() (txs types.Transactions, err error) { - pool.mu.RLock() - defer pool.mu.RUnlock() - - txs = make(types.Transactions, len(pool.pending)) - i := 0 - for _, tx := range pool.pending { - txs[i] = tx - i++ - } - return txs, nil -} - -// Content retrieves the data content of the transaction pool, returning all the -// pending as well as queued transactions, grouped by account and nonce. -func (pool *TxPool) Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { - pool.mu.RLock() - defer pool.mu.RUnlock() - - // Retrieve all the pending transactions and sort by account and by nonce - pending := make(map[common.Address][]*types.Transaction) - for _, tx := range pool.pending { - account, _ := types.Sender(pool.signer, tx) - pending[account] = append(pending[account], tx) - } - // There are no queued transactions in a light pool, just return an empty map - queued := make(map[common.Address][]*types.Transaction) - return pending, queued -} - -// ContentFrom retrieves the data content of the transaction pool, returning the -// pending as well as queued transactions of this address, grouped by nonce. -func (pool *TxPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { - pool.mu.RLock() - defer pool.mu.RUnlock() - - // Retrieve the pending transactions and sort by nonce - var pending []*types.Transaction - for _, tx := range pool.pending { - account, _ := types.Sender(pool.signer, tx) - if account != addr { - continue - } - pending = append(pending, tx) - } - // There are no queued transactions in a light pool, just return an empty map - return pending, []*types.Transaction{} -} - -// RemoveTransactions removes all given transactions from the pool. -func (pool *TxPool) RemoveTransactions(txs types.Transactions) { - pool.mu.Lock() - defer pool.mu.Unlock() - - var hashes []common.Hash - batch := pool.chainDb.NewBatch() - for _, tx := range txs { - hash := tx.Hash() - delete(pool.pending, hash) - batch.Delete(hash.Bytes()) - hashes = append(hashes, hash) - } - batch.Write() - pool.relay.Discard(hashes) -} - -// RemoveTx removes the transaction with the given hash from the pool. -func (pool *TxPool) RemoveTx(hash common.Hash) { - pool.mu.Lock() - defer pool.mu.Unlock() - // delete from pending pool - delete(pool.pending, hash) - pool.chainDb.Delete(hash[:]) - pool.relay.Discard([]common.Hash{hash}) -} diff --git a/light/txpool_test.go b/light/txpool_test.go deleted file mode 100644 index 1eec7bc427..0000000000 --- a/light/txpool_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package light - -import ( - "context" - "math" - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -type testTxRelay struct { - send, discard, mined chan int -} - -func (r *testTxRelay) Send(txs types.Transactions) { - r.send <- len(txs) -} - -func (r *testTxRelay) NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) { - m := len(mined) - if m != 0 { - r.mined <- m - } -} - -func (r *testTxRelay) Discard(hashes []common.Hash) { - r.discard <- len(hashes) -} - -const poolTestTxs = 1000 -const poolTestBlocks = 100 - -// test tx 0..n-1 -var testTx [poolTestTxs]*types.Transaction - -// txs sent before block i -func sentTx(i int) int { - return int(math.Pow(float64(i)/float64(poolTestBlocks), 0.9) * poolTestTxs) -} - -// txs included in block i or before that (minedTx(i) <= sentTx(i)) -func minedTx(i int) int { - return int(math.Pow(float64(i)/float64(poolTestBlocks), 1.1) * poolTestTxs) -} - -func txPoolTestChainGen(i int, block *core.BlockGen) { - s := minedTx(i) - e := minedTx(i + 1) - for i := s; i < e; i++ { - block.AddTx(testTx[i]) - } -} - -func TestTxPool(t *testing.T) { - for i := range testTx { - testTx[i], _ = types.SignTx(types.NewTransaction(uint64(i), acc1Addr, big.NewInt(10000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) - } - - var ( - sdb = rawdb.NewMemoryDatabase() - ldb = rawdb.NewMemoryDatabase() - gspec = &core.Genesis{ - Config: params.TestChainConfig, - Alloc: core.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - ) - // Assemble the test environment - blockchain, _ := core.NewBlockChain(sdb, nil, gspec, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) - _, gchain, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), poolTestBlocks, txPoolTestChainGen) - if _, err := blockchain.InsertChain(gchain); err != nil { - panic(err) - } - - gspec.MustCommit(ldb, trie.NewDatabase(ldb, trie.HashDefaults)) - odr := &testOdr{sdb: sdb, ldb: ldb, serverState: blockchain.StateCache(), indexerConfig: TestClientIndexerConfig} - relay := &testTxRelay{ - send: make(chan int, 1), - discard: make(chan int, 1), - mined: make(chan int, 1), - } - lightchain, _ := NewLightChain(odr, params.TestChainConfig, ethash.NewFullFaker()) - txPermanent = 50 - pool := NewTxPool(params.TestChainConfig, lightchain, relay) - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - - for ii, block := range gchain { - i := ii + 1 - s := sentTx(i - 1) - e := sentTx(i) - for i := s; i < e; i++ { - pool.Add(ctx, testTx[i]) - got := <-relay.send - exp := 1 - if got != exp { - t.Errorf("relay.Send expected len = %d, got %d", exp, got) - } - } - - if _, err := lightchain.InsertHeaderChain([]*types.Header{block.Header()}); err != nil { - panic(err) - } - - got := <-relay.mined - exp := minedTx(i) - minedTx(i-1) - if got != exp { - t.Errorf("relay.NewHead expected len(mined) = %d, got %d", exp, got) - } - - exp = 0 - if i > int(txPermanent)+1 { - exp = minedTx(i-int(txPermanent)-1) - minedTx(i-int(txPermanent)-2) - } - if exp != 0 { - got = <-relay.discard - if got != exp { - t.Errorf("relay.Discard expected len = %d, got %d", exp, got) - } - } - } -} From 00ee953593a5b327061ddc2d568b9b1969a2a914 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 28 Nov 2023 14:40:20 +0330 Subject: [PATCH 07/43] rm comment --- core/state/statedb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index b6f0b17320..8740d7df6f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -60,7 +60,6 @@ type StateLogger interface { OnStorageChange(addr common.Address, slot common.Hash, prev, new common.Hash) OnLog(log *types.Log) // OnNewAccount is called when a new account is created. - // Note: it will be even invoked when precompiled contracts are populated. OnNewAccount(addr common.Address) } From 63979bc9cc461d1f05e6be4646d4dc788de1698f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 28 Nov 2023 13:54:17 +0100 Subject: [PATCH 08/43] cmd/evm, core/state: fix post-exec dump of state (statetests, blockchaintests) (#28504) There were several problems related to dumping state. - If a preimage was missing, even if we had set the `OnlyWithAddresses` to `false`, to export them anyway, the way the mapping was constructed (using `common.Address` as key) made the entries get lost anyway. Concerns both state- and blockchain tests. - Blockchain test execution was not configured to store preimages. This changes makes it so that the block test executor takes a callback, just like the state test executor already does. This callback can be used to examine the post-execution state, e.g. to aid debugging of test failures. --- cmd/evm/blockrunner.go | 9 +++- cmd/geth/chaincmd.go | 5 -- cmd/geth/snapshot.go | 10 ++-- core/state/dump.go | 100 ++++++++++++++++----------------------- core/state/state_test.go | 3 ++ eth/api_debug.go | 18 +++---- eth/api_debug_test.go | 25 +++++----- tests/block_test.go | 8 ++-- tests/block_test_util.go | 13 +++-- 9 files changed, 92 insertions(+), 99 deletions(-) diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index caed9b65fc..c5d836e0ea 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -24,6 +24,7 @@ import ( "regexp" "sort" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/logger" @@ -85,7 +86,13 @@ func blockTestCmd(ctx *cli.Context) error { continue } test := tests[name] - if err := test.Run(false, rawdb.HashScheme, tracer); err != nil { + if err := test.Run(false, rawdb.HashScheme, tracer, func(res error, chain *core.BlockChain) { + if ctx.Bool(DumpFlag.Name) { + if state, _ := chain.State(); state != nil { + fmt.Println(string(state.Dump(nil))) + } + } + }); err != nil { return fmt.Errorf("test %v: %w", name, err) } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 4e528d6502..3b4f516af7 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -473,11 +473,6 @@ func dump(ctx *cli.Context) error { if ctx.Bool(utils.IterativeOutputFlag.Name) { state.IterativeDump(conf, json.NewEncoder(os.Stdout)) } else { - if conf.OnlyWithAddresses { - fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+ - " otherwise the accounts will overwrite each other in the resulting mapping.") - return errors.New("incompatible options") - } fmt.Println(string(state.Dump(conf))) } return nil diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 80d946b894..4284005a02 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -580,11 +580,11 @@ func dumpState(ctx *cli.Context) error { return err } da := &state.DumpAccount{ - Balance: account.Balance.String(), - Nonce: account.Nonce, - Root: account.Root.Bytes(), - CodeHash: account.CodeHash, - SecureKey: accIt.Hash().Bytes(), + Balance: account.Balance.String(), + Nonce: account.Nonce, + Root: account.Root.Bytes(), + CodeHash: account.CodeHash, + AddressHash: accIt.Hash().Bytes(), } if !conf.SkipCode && !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { da.Code = rawdb.ReadCode(db, common.BytesToHash(account.CodeHash)) diff --git a/core/state/dump.go b/core/state/dump.go index 9ce6cd394b..cf46621144 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -49,21 +49,24 @@ type DumpCollector interface { // DumpAccount represents an account in the state. type DumpAccount struct { - Balance string `json:"balance"` - Nonce uint64 `json:"nonce"` - Root hexutil.Bytes `json:"root"` - CodeHash hexutil.Bytes `json:"codeHash"` - Code hexutil.Bytes `json:"code,omitempty"` - Storage map[common.Hash]string `json:"storage,omitempty"` - Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode - SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key + Balance string `json:"balance"` + Nonce uint64 `json:"nonce"` + Root hexutil.Bytes `json:"root"` + CodeHash hexutil.Bytes `json:"codeHash"` + Code hexutil.Bytes `json:"code,omitempty"` + Storage map[common.Hash]string `json:"storage,omitempty"` + Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode + AddressHash hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key } // Dump represents the full dump in a collected format, as one large map. type Dump struct { - Root string `json:"root"` - Accounts map[common.Address]DumpAccount `json:"accounts"` + Root string `json:"root"` + Accounts map[string]DumpAccount `json:"accounts"` + // Next can be set to represent that this dump is only partial, and Next + // is where an iterator should be positioned in order to continue the dump. + Next []byte `json:"next,omitempty"` // nil if no more accounts } // OnRoot implements DumpCollector interface @@ -73,27 +76,11 @@ func (d *Dump) OnRoot(root common.Hash) { // OnAccount implements DumpCollector interface func (d *Dump) OnAccount(addr *common.Address, account DumpAccount) { - if addr != nil { - d.Accounts[*addr] = account + if addr == nil { + d.Accounts[fmt.Sprintf("pre(%s)", account.AddressHash)] = account } -} - -// IteratorDump is an implementation for iterating over data. -type IteratorDump struct { - Root string `json:"root"` - Accounts map[common.Address]DumpAccount `json:"accounts"` - Next []byte `json:"next,omitempty"` // nil if no more accounts -} - -// OnRoot implements DumpCollector interface -func (d *IteratorDump) OnRoot(root common.Hash) { - d.Root = fmt.Sprintf("%x", root) -} - -// OnAccount implements DumpCollector interface -func (d *IteratorDump) OnAccount(addr *common.Address, account DumpAccount) { if addr != nil { - d.Accounts[*addr] = account + d.Accounts[(*addr).String()] = account } } @@ -105,14 +92,14 @@ type iterativeDump struct { // OnAccount implements DumpCollector interface func (d iterativeDump) OnAccount(addr *common.Address, account DumpAccount) { dumpAccount := &DumpAccount{ - Balance: account.Balance, - Nonce: account.Nonce, - Root: account.Root, - CodeHash: account.CodeHash, - Code: account.Code, - Storage: account.Storage, - SecureKey: account.SecureKey, - Address: addr, + Balance: account.Balance, + Nonce: account.Nonce, + Root: account.Root, + CodeHash: account.CodeHash, + Code: account.Code, + Storage: account.Storage, + AddressHash: account.AddressHash, + Address: addr, } d.Encode(dumpAccount) } @@ -150,26 +137,27 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } - account := DumpAccount{ - Balance: data.Balance.String(), - Nonce: data.Nonce, - Root: data.Root[:], - CodeHash: data.CodeHash, - SecureKey: it.Key, - } var ( - addrBytes = s.trie.GetKey(it.Key) - addr = common.BytesToAddress(addrBytes) + account = DumpAccount{ + Balance: data.Balance.String(), + Nonce: data.Nonce, + Root: data.Root[:], + CodeHash: data.CodeHash, + AddressHash: it.Key, + } address *common.Address + addr common.Address + addrBytes = s.trie.GetKey(it.Key) ) if addrBytes == nil { - // Preimage missing missingPreimages++ if conf.OnlyWithAddresses { continue } } else { + addr = common.BytesToAddress(addrBytes) address = &addr + account.Address = address } obj := newObject(s, addr, &data) if !conf.SkipCode { @@ -220,12 +208,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] return nextKey } -// RawDump returns the entire state an a single large object +// RawDump returns the state. If the processing is aborted e.g. due to options +// reaching Max, the `Next` key is set on the returned Dump. func (s *StateDB) RawDump(opts *DumpConfig) Dump { dump := &Dump{ - Accounts: make(map[common.Address]DumpAccount), + Accounts: make(map[string]DumpAccount), } - s.DumpToCollector(dump, opts) + dump.Next = s.DumpToCollector(dump, opts) return *dump } @@ -234,7 +223,7 @@ func (s *StateDB) Dump(opts *DumpConfig) []byte { dump := s.RawDump(opts) json, err := json.MarshalIndent(dump, "", " ") if err != nil { - fmt.Println("Dump err", err) + log.Error("Error dumping state", "err", err) } return json } @@ -243,12 +232,3 @@ func (s *StateDB) Dump(opts *DumpConfig) []byte { func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) { s.DumpToCollector(iterativeDump{output}, opts) } - -// IteratorDump dumps out a batch of accounts starts with the given start key -func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump { - iterator := &IteratorDump{ - Accounts: make(map[common.Address]DumpAccount), - } - iterator.Next = s.DumpToCollector(iterator, opts) - return *iterator -} diff --git a/core/state/state_test.go b/core/state/state_test.go index 2553133dea..2f45ba44b4 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -71,6 +71,7 @@ func TestDump(t *testing.T) { "nonce": 0, "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "address": "0x0000000000000000000000000000000000000001", "key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d" }, "0x0000000000000000000000000000000000000002": { @@ -78,6 +79,7 @@ func TestDump(t *testing.T) { "nonce": 0, "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "address": "0x0000000000000000000000000000000000000002", "key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62" }, "0x0000000000000000000000000000000000000102": { @@ -86,6 +88,7 @@ func TestDump(t *testing.T) { "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "0x87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", "code": "0x03030303030303", + "address": "0x0000000000000000000000000000000000000102", "key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1" } } diff --git a/eth/api_debug.go b/eth/api_debug.go index dc9f568146..05010a3969 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -133,7 +133,7 @@ func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) const AccountRangeMaxResults = 256 // AccountRange enumerates all accounts in the given block and start point in paging request -func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hexutil.Bytes, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { +func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hexutil.Bytes, maxResults int, nocode, nostorage, incompletes bool) (state.Dump, error) { var stateDb *state.StateDB var err error @@ -144,7 +144,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex // the miner and operate on those _, stateDb = api.eth.miner.Pending() if stateDb == nil { - return state.IteratorDump{}, errors.New("pending state is not available") + return state.Dump{}, errors.New("pending state is not available") } } else { var header *types.Header @@ -158,29 +158,29 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex default: block := api.eth.blockchain.GetBlockByNumber(uint64(number)) if block == nil { - return state.IteratorDump{}, fmt.Errorf("block #%d not found", number) + return state.Dump{}, fmt.Errorf("block #%d not found", number) } header = block.Header() } if header == nil { - return state.IteratorDump{}, fmt.Errorf("block #%d not found", number) + return state.Dump{}, fmt.Errorf("block #%d not found", number) } stateDb, err = api.eth.BlockChain().StateAt(header.Root) if err != nil { - return state.IteratorDump{}, err + return state.Dump{}, err } } } else if hash, ok := blockNrOrHash.Hash(); ok { block := api.eth.blockchain.GetBlockByHash(hash) if block == nil { - return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex()) + return state.Dump{}, fmt.Errorf("block %s not found", hash.Hex()) } stateDb, err = api.eth.BlockChain().StateAt(block.Root()) if err != nil { - return state.IteratorDump{}, err + return state.Dump{}, err } } else { - return state.IteratorDump{}, errors.New("either block number or block hash must be specified") + return state.Dump{}, errors.New("either block number or block hash must be specified") } opts := &state.DumpConfig{ @@ -193,7 +193,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex if maxResults > AccountRangeMaxResults || maxResults <= 0 { opts.Max = AccountRangeMaxResults } - return stateDb.IteratorDump(opts), nil + return stateDb.RawDump(opts), nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index 3d3444a871..184b90dd09 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -21,6 +21,7 @@ import ( "fmt" "math/big" "reflect" + "strings" "testing" "github.com/davecgh/go-spew/spew" @@ -35,8 +36,8 @@ import ( var dumper = spew.ConfigState{Indent: " "} -func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { - result := statedb.IteratorDump(&state.DumpConfig{ +func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.Dump { + result := statedb.RawDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, OnlyWithAddresses: false, @@ -47,12 +48,12 @@ func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, st if len(result.Accounts) != expectedNum { t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) } - for address := range result.Accounts { - if address == (common.Address{}) { - t.Fatalf("empty address returned") + for addr, acc := range result.Accounts { + if strings.HasSuffix(addr, "pre") || acc.Address == nil { + t.Fatalf("account without prestate (address) returned: %v", addr) } - if !statedb.Exist(address) { - t.Fatalf("account not found in state %s", address.Hex()) + if !statedb.Exist(*acc.Address) { + t.Fatalf("account not found in state %s", acc.Address.Hex()) } } return result @@ -92,16 +93,16 @@ func TestAccountRange(t *testing.T) { secondResult := accountRangeTest(t, &trie, sdb, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) hList := make([]common.Hash, 0) - for addr1 := range firstResult.Accounts { - // If address is empty, then it makes no sense to compare + for addr1, acc := range firstResult.Accounts { + // If address is non-available, then it makes no sense to compare // them as they might be two different accounts. - if addr1 == (common.Address{}) { + if acc.Address == nil { continue } if _, duplicate := secondResult.Accounts[addr1]; duplicate { t.Fatalf("pagination test failed: results should not overlap") } - hList = append(hList, crypto.Keccak256Hash(addr1.Bytes())) + hList = append(hList, crypto.Keccak256Hash(acc.Address.Bytes())) } // Test to see if it's possible to recover from the middle of the previous // set and get an even split between the first and second sets. @@ -140,7 +141,7 @@ func TestEmptyAccountRange(t *testing.T) { st.Commit(0, true) st, _ = state.New(types.EmptyRootHash, statedb, nil) - results := st.IteratorDump(&state.DumpConfig{ + results := st.RawDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, OnlyWithAddresses: true, diff --git a/tests/block_test.go b/tests/block_test.go index e913ecbc90..aa6f27b8f3 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -74,19 +74,19 @@ func TestExecutionSpec(t *testing.T) { } func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) { - if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil)); err != nil { + if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { t.Errorf("test in hash mode without snapshotter failed: %v", err) return } - if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil)); err != nil { + if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { t.Errorf("test in hash mode with snapshotter failed: %v", err) return } - if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil)); err != nil { + if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { t.Errorf("test in path mode without snapshotter failed: %v", err) return } - if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil)); err != nil { + if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { t.Errorf("test in path mode with snapshotter failed: %v", err) return } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index d7409f8000..e0130be48a 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -108,7 +108,7 @@ type btHeaderMarshaling struct { ExcessBlobGas *math.HexOrDecimal64 } -func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) error { +func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) (result error) { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} @@ -116,7 +116,9 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) er // import pre accounts & construct test genesis block & state root var ( db = rawdb.NewMemoryDatabase() - tconf = &trie.Config{} + tconf = &trie.Config{ + Preimages: true, + } ) if scheme == rawdb.PathScheme { tconf.PathDB = pathdb.Defaults @@ -141,7 +143,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) er // Wrap the original engine within the beacon-engine engine := beacon.New(ethash.NewFaker()) - cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme} + cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme, Preimages: true} if snapshotter { cache.SnapshotLimit = 1 cache.SnapshotWait = true @@ -158,6 +160,11 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger) er if err != nil { return err } + // Import succeeded: regardless of whether the _test_ succeeds or not, schedule + // the post-check to run + if postCheck != nil { + defer postCheck(result, chain) + } cmlast := chain.CurrentBlock().Hash() if common.Hash(t.json.BestBlock) != cmlast { return fmt.Errorf("last block hash validation mismatch: want: %x, have: %x", t.json.BestBlock, cmlast) From 248dc50ee81310a2ad45d52f091c70b6c1ecaa35 Mon Sep 17 00:00:00 2001 From: Shivam Sandbhor Date: Tue, 28 Nov 2023 19:19:08 +0530 Subject: [PATCH 09/43] ethereum: remove TODO comment about subscription (#28609) --- interfaces.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/interfaces.go b/interfaces.go index eb9af60076..c4948191d1 100644 --- a/interfaces.go +++ b/interfaces.go @@ -29,8 +29,6 @@ import ( // NotFound is returned by API methods if the requested item does not exist. var NotFound = errors.New("not found") -// TODO: move subscription to package event - // Subscription represents an event subscription where events are // delivered on a data channel. type Subscription interface { From bbc5db840553a39141e1a8630181ff37a9cc85d8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 28 Nov 2023 17:30:35 +0330 Subject: [PATCH 10/43] eth/tracers/js: fix type inconsistencies (#28488) This change fixes two type-inconsistencies in the JS tracer: - In most places we return byte arrays as a `Uint8Array` to the tracer. However it seems we missed doing the conversion for `ctx` fields which are passed to the tracer during `result`. They are passed as simple arrays. I think Uint8Arrays are more suitable and we should change this inconsistency. Note: this will be a breaking-change. But I believe the effect is small. If we look at our tracers we see that these fields (`ctx.from`, `ctx.to`, etc.) are used in 2 ways. Passed to `toHex` which takes both array or buffer. Or the length was measured which is the same for both types. - The `slice` taking in `int, int` params versus `memory.slice` taking `int64, int64` params. I suggest changing `slice` types to `int64`. This should have no effect almost in any case. --- eth/tracers/js/goja.go | 58 +++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index d22d140988..07c138bae4 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -142,19 +142,29 @@ func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracer vm: vm, ctx: make(map[string]goja.Value), } + + t.setTypeConverters() + t.setBuiltinFunctions() + if ctx == nil { ctx = new(tracers.Context) } if ctx.BlockHash != (common.Hash{}) { - t.ctx["blockHash"] = vm.ToValue(ctx.BlockHash.Bytes()) + blockHash, err := t.toBuf(vm, ctx.BlockHash.Bytes()) + if err != nil { + return nil, err + } + t.ctx["blockHash"] = blockHash if ctx.TxHash != (common.Hash{}) { t.ctx["txIndex"] = vm.ToValue(ctx.TxIndex) - t.ctx["txHash"] = vm.ToValue(ctx.TxHash.Bytes()) + txHash, err := t.toBuf(vm, ctx.TxHash.Bytes()) + if err != nil { + return nil, err + } + t.ctx["txHash"] = txHash } } - t.setTypeConverters() - t.setBuiltinFunctions() ret, err := vm.RunString("(" + code + ")") if err != nil { return nil, err @@ -224,6 +234,10 @@ func (t *jsTracer) CaptureTxEnd(restGas uint64) { // CaptureStart implements the Tracer interface to initialize the tracing operation. func (t *jsTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) { + cancel := func(err error) { + t.err = err + t.env.Cancel() + } t.env = env db := &dbObj{db: env.StateDB, vm: t.vm, toBig: t.toBig, toBuf: t.toBuf, fromBuf: t.fromBuf} t.dbValue = db.setupObject() @@ -232,19 +246,34 @@ func (t *jsTracer) CaptureStart(env *vm.EVM, from common.Address, to common.Addr } else { t.ctx["type"] = t.vm.ToValue("CALL") } - t.ctx["from"] = t.vm.ToValue(from.Bytes()) - t.ctx["to"] = t.vm.ToValue(to.Bytes()) - t.ctx["input"] = t.vm.ToValue(input) + fromVal, err := t.toBuf(t.vm, from.Bytes()) + if err != nil { + cancel(err) + return + } + t.ctx["from"] = fromVal + toVal, err := t.toBuf(t.vm, to.Bytes()) + if err != nil { + cancel(err) + return + } + t.ctx["to"] = toVal + inputVal, err := t.toBuf(t.vm, input) + if err != nil { + cancel(err) + return + } + t.ctx["input"] = inputVal t.ctx["gas"] = t.vm.ToValue(t.gasLimit) gasPriceBig, err := t.toBig(t.vm, env.TxContext.GasPrice.String()) if err != nil { - t.err = err + cancel(err) return } t.ctx["gasPrice"] = gasPriceBig valueBig, err := t.toBig(t.vm, value.String()) if err != nil { - t.err = err + cancel(err) return } t.ctx["value"] = valueBig @@ -293,10 +322,15 @@ func (t *jsTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, scope // CaptureEnd is called after the call finishes to finalize the tracing. func (t *jsTracer) CaptureEnd(output []byte, gasUsed uint64, err error) { - t.ctx["output"] = t.vm.ToValue(output) if err != nil { t.ctx["error"] = t.vm.ToValue(err.Error()) } + outputVal, err := t.toBuf(t.vm, output) + if err != nil { + t.err = err + return + } + t.ctx["output"] = outputVal } // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). @@ -465,13 +499,13 @@ func (t *jsTracer) setBuiltinFunctions() { } return false }) - vm.Set("slice", func(slice goja.Value, start, end int) goja.Value { + vm.Set("slice", func(slice goja.Value, start, end int64) goja.Value { b, err := t.fromBuf(vm, slice, false) if err != nil { vm.Interrupt(err) return nil } - if start < 0 || start > end || end > len(b) { + if start < 0 || start > end || end > int64(len(b)) { vm.Interrupt(fmt.Sprintf("Tracer accessed out of bound memory: available %d, offset %d, size %d", len(b), start, end-start)) return nil } From 34dcd749355f161f1ed7efe9e59c6c9d13402ff6 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Nov 2023 19:16:50 +0100 Subject: [PATCH 11/43] crypto/secp256k1: fix 32-bit tests when CGO_ENABLED=0 (#28602) --- crypto/secp256k1/secp256_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/secp256k1/secp256_test.go b/crypto/secp256k1/secp256_test.go index ef2a3a3790..74408d06d2 100644 --- a/crypto/secp256k1/secp256_test.go +++ b/crypto/secp256k1/secp256_test.go @@ -2,6 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. +//go:build !gofuzz && cgo +// +build !gofuzz,cgo + package secp256k1 import ( From e0c7ad01abb325e443e63ca15e09282bcd1b6909 Mon Sep 17 00:00:00 2001 From: Jakub Freebit <49676311+jakub-freebit@users.noreply.github.com> Date: Wed, 29 Nov 2023 05:29:00 +0900 Subject: [PATCH 12/43] consensus: verify the nonexistence of shanghai- and cancun-specific header fields (#28605) --- consensus/clique/clique.go | 22 ++++++++++++++++++++++ consensus/ethash/consensus.go | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index f708050abd..c693189ea5 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -302,9 +302,22 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H if chain.Config().IsShanghai(header.Number, header.Time) { return errors.New("clique does not support shanghai fork") } + // Verify the non-existence of withdrawalsHash. + if header.WithdrawalsHash != nil { + return fmt.Errorf("invalid withdrawalsHash: have %x, expected nil", header.WithdrawalsHash) + } if chain.Config().IsCancun(header.Number, header.Time) { return errors.New("clique does not support cancun fork") } + // Verify the non-existence of cancun-specific header fields + switch { + case header.ExcessBlobGas != nil: + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + case header.BlobGasUsed != nil: + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + case header.ParentBeaconRoot != nil: + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + } // All basic checks passed, verify cascading fields return c.verifyCascadingFields(chain, header, parents) } @@ -753,6 +766,15 @@ func encodeSigHeader(w io.Writer, header *types.Header) { if header.WithdrawalsHash != nil { panic("unexpected withdrawal hash value in clique") } + if header.ExcessBlobGas != nil { + panic("unexpected excess blob gas value in clique") + } + if header.BlobGasUsed != nil { + panic("unexpected blob gas used value in clique") + } + if header.ParentBeaconRoot != nil { + panic("unexpected parent beacon root value in clique") + } if err := rlp.Encode(w, enc); err != nil { panic("can't encode: " + err.Error()) } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 8eb9863da1..130dfdf213 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -266,9 +266,22 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa if chain.Config().IsShanghai(header.Number, header.Time) { return errors.New("ethash does not support shanghai fork") } + // Verify the non-existence of withdrawalsHash. + if header.WithdrawalsHash != nil { + return fmt.Errorf("invalid withdrawalsHash: have %x, expected nil", header.WithdrawalsHash) + } if chain.Config().IsCancun(header.Number, header.Time) { return errors.New("ethash does not support cancun fork") } + // Verify the non-existence of cancun-specific header fields + switch { + case header.ExcessBlobGas != nil: + return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas) + case header.BlobGasUsed != nil: + return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed) + case header.ParentBeaconRoot != nil: + return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot) + } // Add some fake checks for tests if ethash.fakeDelay != nil { time.Sleep(*ethash.fakeDelay) @@ -533,6 +546,15 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { if header.WithdrawalsHash != nil { panic("withdrawal hash set on ethash") } + if header.ExcessBlobGas != nil { + panic("excess blob gas set on ethash") + } + if header.BlobGasUsed != nil { + panic("blob gas used set on ethash") + } + if header.ParentBeaconRoot != nil { + panic("parent beacon root set on ethash") + } rlp.Encode(hasher, enc) hasher.Sum(hash[:0]) return hash From 61b844f2b2cf6b9233a5bcd43c68d9be6f7392b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 28 Nov 2023 22:31:47 +0200 Subject: [PATCH 13/43] eth/gasestimator: allow slight estimation error in favor of less iterations (#28618) * eth/gasestimator: early exit for plain transfer and error allowance * core, eth/gasestimator: hard guess at a possible required gas * internal/ethapi: update estimation tests with the error ratio * eth/gasestimator: I hate you linter * graphql: fix gas estimation test --------- Co-authored-by: Oren --- core/state_transition.go | 23 +++++++++------- eth/gasestimator/gasestimator.go | 45 ++++++++++++++++++++++++++++++-- graphql/graphql_test.go | 2 +- internal/ethapi/api.go | 13 ++++++--- internal/ethapi/api_test.go | 2 +- 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 612fdd7813..540f63fda7 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -32,9 +32,10 @@ import ( // ExecutionResult includes all output after executing given evm // message no matter the execution itself is successful or not. type ExecutionResult struct { - UsedGas uint64 // Total used gas but include the refunded gas - Err error // Any error encountered during the execution(listed in core/vm/errors.go) - ReturnData []byte // Returned data from evm(function result or data supplied with revert opcode) + UsedGas uint64 // Total used gas, not including the refunded gas + RefundedGas uint64 // Total gas refunded after execution + Err error // Any error encountered during the execution(listed in core/vm/errors.go) + ReturnData []byte // Returned data from evm(function result or data supplied with revert opcode) } // Unwrap returns the internal evm error which allows us for further @@ -419,12 +420,13 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, msg.Value) } + var gasRefund uint64 if !rules.IsLondon { // Before EIP-3529: refunds were capped to gasUsed / 2 - st.refundGas(params.RefundQuotient) + gasRefund = st.refundGas(params.RefundQuotient) } else { // After EIP-3529: refunds are capped to gasUsed / 5 - st.refundGas(params.RefundQuotientEIP3529) + gasRefund = st.refundGas(params.RefundQuotientEIP3529) } effectiveTip := msg.GasPrice if rules.IsLondon { @@ -442,13 +444,14 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { } return &ExecutionResult{ - UsedGas: st.gasUsed(), - Err: vmerr, - ReturnData: ret, + UsedGas: st.gasUsed(), + RefundedGas: gasRefund, + Err: vmerr, + ReturnData: ret, }, nil } -func (st *StateTransition) refundGas(refundQuotient uint64) { +func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Apply refund counter, capped to a refund quotient refund := st.gasUsed() / refundQuotient if refund > st.state.GetRefund() { @@ -463,6 +466,8 @@ func (st *StateTransition) refundGas(refundQuotient uint64) { // Also return remaining gas to the block gas counter so it is // available for the next transaction. st.gp.AddGas(st.gasRemaining) + + return refund } // gasUsed returns the amount of gas used up by the state transition. diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index 3e74b5b08b..4a8e20dfed 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -42,6 +42,8 @@ type Options struct { Chain core.ChainContext // Chain context to access past block hashes Header *types.Header // Header defining the block context to execute in State *state.StateDB // Pre-state on top of which to estimate the gas + + ErrorRatio float64 // Allowed overestimation ratio for faster estimation termination } // Estimate returns the lowest possible gas limit that allows the transaction to @@ -86,16 +88,28 @@ func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uin if transfer == nil { transfer = new(big.Int) } - log.Warn("Gas estimation capped by limited funds", "original", hi, "balance", balance, + log.Debug("Gas estimation capped by limited funds", "original", hi, "balance", balance, "sent", transfer, "maxFeePerGas", feeCap, "fundable", allowance) hi = allowance.Uint64() } } // Recap the highest gas allowance with specified gascap. if gasCap != 0 && hi > gasCap { - log.Warn("Caller gas above allowance, capping", "requested", hi, "cap", gasCap) + log.Debug("Caller gas above allowance, capping", "requested", hi, "cap", gasCap) hi = gasCap } + // If the transaction is a plain value transfer, short circuit estimation and + // directly try 21000. Returning 21000 without any execution is dangerous as + // some tx field combos might bump the price up even for plain transfers (e.g. + // unused access list items). Ever so slightly wasteful, but safer overall. + if len(call.Data) == 0 { + if call.To != nil && opts.State.GetCodeSize(*call.To) == 0 { + failed, _, err := execute(ctx, call, opts, params.TxGas) + if !failed && err == nil { + return params.TxGas, nil, nil + } + } + } // We first execute the transaction at the highest allowable gas limit, since if this fails we // can return error immediately. failed, result, err := execute(ctx, call, opts, hi) @@ -115,8 +129,35 @@ func Estimate(ctx context.Context, call *core.Message, opts *Options, gasCap uin // limit for these cases anyway. lo = result.UsedGas - 1 + // There's a fairly high chance for the transaction to execute successfully + // with gasLimit set to the first execution's usedGas + gasRefund. Explicitly + // check that gas amount and use as a limit for the binary search. + optimisticGasLimit := (result.UsedGas + result.RefundedGas + params.CallStipend) * 64 / 63 + if optimisticGasLimit < hi { + failed, _, err = execute(ctx, call, opts, optimisticGasLimit) + if err != nil { + // This should not happen under normal conditions since if we make it this far the + // transaction had run without error at least once before. + log.Error("Execution error in estimate gas", "err", err) + return 0, nil, err + } + if failed { + lo = optimisticGasLimit + } else { + hi = optimisticGasLimit + } + } // Binary search for the smallest gas limit that allows the tx to execute successfully. for lo+1 < hi { + if opts.ErrorRatio > 0 { + // It is a bit pointless to return a perfect estimation, as changing + // network conditions require the caller to bump it up anyway. Since + // wallets tend to use 20-25% bump, allowing a small approximation + // error is fine (as long as it's upwards). + if float64(hi-lo)/float64(hi) < opts.ErrorRatio { + break + } + } mid := (hi + lo) / 2 if mid > lo*2 { // Most txs don't need much higher gas limit than their gas used, and most txs don't diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index a83d6bbd46..f91229d015 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -139,7 +139,7 @@ func TestGraphQLBlockSerialization(t *testing.T) { // should return `estimateGas` as decimal { body: `{"query": "{block{ estimateGas(data:{}) }}"}`, - want: `{"data":{"block":{"estimateGas":"0xcf08"}}}`, + want: `{"data":{"block":{"estimateGas":"0xd221"}}}`, code: 200, }, // should return `status` as decimal diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f322132769..c0b28e4b69 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -51,6 +51,10 @@ import ( "github.com/tyler-smith/go-bip39" ) +// estimateGasErrorRatio is the amount of overestimation eth_estimateGas is +// allowed to produce in order to speed up calculations. +const estimateGasErrorRatio = 0.015 + // EthereumAPI provides an API to access Ethereum related information. type EthereumAPI struct { b Backend @@ -1189,10 +1193,11 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } // Construct the gas estimator option from the user input opts := &gasestimator.Options{ - Config: b.ChainConfig(), - Chain: NewChainContext(ctx, b), - Header: header, - State: state, + Config: b.ChainConfig(), + Chain: NewChainContext(ctx, b), + Header: header, + State: state, + ErrorRatio: estimateGasErrorRatio, } // Run the gas estimation andwrap any revertals into a custom return call, err := args.ToMessage(gasCap, header.BaseFee) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 9b08fd8d42..c2490ac703 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -735,7 +735,7 @@ func TestEstimateGas(t *testing.T) { t.Errorf("test %d: want no error, have %v", i, err) continue } - if uint64(result) != tc.want { + if float64(result) > float64(tc.want)*(1+estimateGasErrorRatio) { t.Errorf("test %d, result mismatch, have\n%v\n, want\n%v\n", i, uint64(result), tc.want) } } From 28e73717016cdc9ebdb5fdb3474cfbd3bd2d2524 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Wed, 29 Nov 2023 15:33:50 +0800 Subject: [PATCH 14/43] all: replace log15 with slog (#28187) This PR replaces Geth's logger package (a fork of [log15](https://github.com/inconshreveable/log15)) with an implementation using slog, a logging library included as part of the Go standard library as of Go1.21. Main changes are as follows: * removes any log handlers that were unused in the Geth codebase. * Json, logfmt, and terminal formatters are now slog handlers. * Verbosity level constants are changed to match slog constant values. Internal translation is done to make this opaque to the user and backwards compatible with existing `--verbosity` and `--vmodule` options. * `--log.backtraceat` and `--log.debug` are removed. The external-facing API is largely the same as the existing Geth logger. Logger method signatures remain unchanged. A small semantic difference is that a `Handler` can only be set once per `Logger` and not changed dynamically. This just means that a new logger must be instantiated every time the handler of the root logger is changed. ---- For users of the `go-ethereum/log` module. If you were using this module for your own project, you will need to change the initialization. If you previously did ```golang log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) ``` You now instead need to do ```golang log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) ``` See more about reasoning here: https://github.com/ethereum/go-ethereum/issues/28558#issuecomment-1820606613 --- cmd/abigen/main.go | 2 +- cmd/bootnode/main.go | 7 +- cmd/clef/main.go | 3 +- cmd/devp2p/runtest.go | 2 +- cmd/evm/internal/t8ntool/block.go | 7 +- cmd/evm/internal/t8ntool/transaction.go | 7 +- cmd/evm/internal/t8ntool/transition.go | 8 +- cmd/geth/logging_test.go | 48 ++ cmd/geth/logtestcmd_active.go | 5 +- cmd/geth/main.go | 2 + cmd/geth/testdata/logging/logtest-json.txt | 100 ++-- cmd/geth/testdata/logging/logtest-logfmt.txt | 102 ++-- .../testdata/logging/logtest-terminal.txt | 102 ++-- cmd/utils/flags.go | 7 + cmd/utils/flags_legacy.go | 14 + core/state/snapshot/generate_test.go | 2 +- core/txpool/blobpool/blobpool_test.go | 10 +- eth/catalyst/api_test.go | 2 +- eth/downloader/queue_test.go | 4 +- go.mod | 17 +- go.sum | 34 +- internal/debug/api.go | 9 +- internal/debug/flags.go | 116 ++-- internal/testlog/testlog.go | 126 +++-- log/CONTRIBUTORS | 11 - log/LICENSE | 13 - log/README.md | 77 --- log/README_ETHEREUM.md | 5 - log/doc.go | 327 ------------ log/format.go | 337 +++--------- log/handler.go | 504 ++++++------------ log/handler_glog.go | 155 +++--- log/logger.go | 380 ++++++------- log/logger_test.go | 62 +-- log/root.go | 49 +- log/syslog.go | 58 -- miner/stress/clique/main.go | 2 +- p2p/discover/v4_udp_test.go | 7 +- p2p/discover/v5_udp_test.go | 7 +- p2p/simulations/adapters/exec.go | 20 +- p2p/simulations/adapters/types.go | 5 +- p2p/simulations/examples/ping-pong.go | 2 +- p2p/simulations/http_test.go | 4 +- signer/core/auditlog.go | 9 +- signer/storage/aes_gcm_storage_test.go | 5 +- 45 files changed, 971 insertions(+), 1804 deletions(-) delete mode 100644 log/CONTRIBUTORS delete mode 100644 log/LICENSE delete mode 100644 log/README.md delete mode 100644 log/README_ETHEREUM.md delete mode 100644 log/doc.go delete mode 100644 log/syslog.go diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 221f45c078..0149dec527 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -232,7 +232,7 @@ func abigen(c *cli.Context) error { } func main() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) if err := app.Run(os.Args); err != nil { fmt.Fprintln(os.Stderr, err) diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 5c1635de39..1660b43b74 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" + "golang.org/x/exp/slog" ) func main() { @@ -52,10 +53,10 @@ func main() { ) flag.Parse() - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(*verbosity)) + glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) + glogger.Verbosity(slog.Level(*verbosity)) glogger.Vmodule(*vmodule) - log.Root().SetHandler(glogger) + log.SetDefault(log.NewLogger(glogger)) natm, err := nat.Parse(*natdesc) if err != nil { diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 63f34effb7..27b7b70771 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -57,6 +57,7 @@ import ( "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" + "golang.org/x/exp/slog" ) const legalWarning = ` @@ -492,7 +493,7 @@ func initialize(c *cli.Context) error { if usecolor { output = colorable.NewColorable(logOutput) } - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(c.Int(logLevelFlag.Name)), log.StreamHandler(output, log.TerminalFormat(usecolor)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, slog.Level(c.Int(logLevelFlag.Name)), usecolor))) return nil } diff --git a/cmd/devp2p/runtest.go b/cmd/devp2p/runtest.go index f72aa91119..76af53ee4d 100644 --- a/cmd/devp2p/runtest.go +++ b/cmd/devp2p/runtest.go @@ -54,7 +54,7 @@ func runTests(ctx *cli.Context, tests []utesting.Test) error { } // Disable logging unless explicitly enabled. if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") { - log.Root().SetHandler(log.DiscardHandler()) + log.SetDefault(log.NewLogger(log.DiscardHandler())) } // Run the tests. var run = utesting.RunTests diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 5c0e28e284..429ae12c54 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/urfave/cli/v2" + "golang.org/x/exp/slog" ) //go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go @@ -216,9 +217,9 @@ func (i *bbInput) sealClique(block *types.Block) (*types.Block, error) { // BuildBlock constructs a block from the given inputs. func BuildBlock(ctx *cli.Context) error { // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) - log.Root().SetHandler(glogger) + glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) + glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) + log.SetDefault(log.NewLogger(glogger)) baseDir, err := createBasedir(ctx) if err != nil { diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 03a2e2eb99..e1c98c7fe2 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" + "golang.org/x/exp/slog" ) type result struct { @@ -66,9 +67,9 @@ func (r *result) MarshalJSON() ([]byte, error) { func Transaction(ctx *cli.Context) error { // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) - log.Root().SetHandler(glogger) + glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) + glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) + log.SetDefault(log.NewLogger(glogger)) var ( err error diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index d517592e5c..a01dfedab9 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -24,6 +24,8 @@ import ( "os" "path" + "golang.org/x/exp/slog" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/misc/eip1559" @@ -81,9 +83,9 @@ type input struct { func Transition(ctx *cli.Context) error { // Configure the go-ethereum logger - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) - glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) - log.Root().SetHandler(glogger) + glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) + glogger.Verbosity(slog.Level(ctx.Int(VerbosityFlag.Name))) + log.SetDefault(log.NewLogger(glogger)) var ( err error diff --git a/cmd/geth/logging_test.go b/cmd/geth/logging_test.go index 69fe0fadf7..50991554b4 100644 --- a/cmd/geth/logging_test.go +++ b/cmd/geth/logging_test.go @@ -28,6 +28,7 @@ import ( "os/exec" "strings" "testing" + "encoding/json" "github.com/ethereum/go-ethereum/internal/reexec" ) @@ -98,6 +99,53 @@ func testConsoleLogging(t *testing.T, format string, tStart, tEnd int) { } } +func TestJsonLogging(t *testing.T) { + t.Parallel() + haveB, err := runSelf("--log.format", "json", "logtest") + if err != nil { + t.Fatal(err) + } + readFile, err := os.Open("testdata/logging/logtest-json.txt") + if err != nil { + t.Fatal(err) + } + wantLines := split(readFile) + haveLines := split(bytes.NewBuffer(haveB)) + for i, wantLine := range wantLines { + if i > len(haveLines)-1 { + t.Fatalf("format %v, line %d missing, want:%v", "json", i, wantLine) + } + haveLine := haveLines[i] + for strings.Contains(haveLine, "Unknown config environment variable") { + // This can happen on CI runs. Drop it. + haveLines = append(haveLines[:i], haveLines[i+1:]...) + haveLine = haveLines[i] + } + var have, want []byte + { + var h map[string]any + if err := json.Unmarshal([]byte(haveLine), &h); err != nil { + t.Fatal(err) + } + h["t"] = "xxx" + have, _ = json.Marshal(h) + } + { + var w map[string]any + if err := json.Unmarshal([]byte(wantLine), &w); err != nil { + t.Fatal(err) + } + w["t"] = "xxx" + want, _ = json.Marshal(w) + } + if !bytes.Equal(have, want) { + // show an intelligent diff + t.Logf(nicediff(have, want)) + t.Errorf("file content wrong") + } + } +} + func TestVmodule(t *testing.T) { t.Parallel() checkOutput := func(level int, want, wantNot string) { diff --git a/cmd/geth/logtestcmd_active.go b/cmd/geth/logtestcmd_active.go index 0632f9ca4b..0ca4cc621d 100644 --- a/cmd/geth/logtestcmd_active.go +++ b/cmd/geth/logtestcmd_active.go @@ -26,6 +26,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/log" "github.com/holiman/uint256" "github.com/urfave/cli/v2" @@ -49,7 +50,9 @@ func (c customQuotedStringer) String() string { // logTest is an entry point which spits out some logs. This is used by testing // to verify expected outputs func logTest(ctx *cli.Context) error { - log.ResetGlobalState() + // clear field padding map + debug.ResetLogging() + { // big.Int ba, _ := new(big.Int).SetString("111222333444555678999", 10) // "111,222,333,444,555,678,999" bb, _ := new(big.Int).SetString("-111222333444555678999", 10) // "-111,222,333,444,555,678,999" diff --git a/cmd/geth/main.go b/cmd/geth/main.go index e5a17e45cf..0d5939bd20 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -144,6 +144,8 @@ var ( utils.GpoMaxGasPriceFlag, utils.GpoIgnoreGasPriceFlag, configFileFlag, + utils.LogDebugFlag, + utils.LogBacktraceAtFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ diff --git a/cmd/geth/testdata/logging/logtest-json.txt b/cmd/geth/testdata/logging/logtest-json.txt index 6cb2476dbd..bdc1ae4de6 100644 --- a/cmd/geth/testdata/logging/logtest-json.txt +++ b/cmd/geth/testdata/logging/logtest-json.txt @@ -1,49 +1,51 @@ -{"111,222,333,444,555,678,999":"111222333444555678999","lvl":"info","msg":"big.Int","t":"2023-11-09T08:33:19.464383209+01:00"} -{"-111,222,333,444,555,678,999":"-111222333444555678999","lvl":"info","msg":"-big.Int","t":"2023-11-09T08:33:19.46455928+01:00"} -{"11,122,233,344,455,567,899,900":"11122233344455567899900","lvl":"info","msg":"big.Int","t":"2023-11-09T08:33:19.464582073+01:00"} -{"-11,122,233,344,455,567,899,900":"-11122233344455567899900","lvl":"info","msg":"-big.Int","t":"2023-11-09T08:33:19.464594846+01:00"} -{"111,222,333,444,555,678,999":"0x607851afc94ca2517","lvl":"info","msg":"uint256","t":"2023-11-09T08:33:19.464607873+01:00"} -{"11,122,233,344,455,567,899,900":"0x25aeffe8aaa1ef67cfc","lvl":"info","msg":"uint256","t":"2023-11-09T08:33:19.464694639+01:00"} -{"1,000,000":1000000,"lvl":"info","msg":"int64","t":"2023-11-09T08:33:19.464708835+01:00"} -{"-1,000,000":-1000000,"lvl":"info","msg":"int64","t":"2023-11-09T08:33:19.464725054+01:00"} -{"9,223,372,036,854,775,807":9223372036854775807,"lvl":"info","msg":"int64","t":"2023-11-09T08:33:19.464735773+01:00"} -{"-9,223,372,036,854,775,808":-9223372036854775808,"lvl":"info","msg":"int64","t":"2023-11-09T08:33:19.464744532+01:00"} -{"1,000,000":1000000,"lvl":"info","msg":"uint64","t":"2023-11-09T08:33:19.464752807+01:00"} -{"18,446,744,073,709,551,615":18446744073709551615,"lvl":"info","msg":"uint64","t":"2023-11-09T08:33:19.464779296+01:00"} -{"key":"special \r\n\t chars","lvl":"info","msg":"Special chars in value","t":"2023-11-09T08:33:19.464794181+01:00"} -{"lvl":"info","msg":"Special chars in key","special \n\t chars":"value","t":"2023-11-09T08:33:19.464827197+01:00"} -{"lvl":"info","msg":"nospace","nospace":"nospace","t":"2023-11-09T08:33:19.464841118+01:00"} -{"lvl":"info","msg":"with space","t":"2023-11-09T08:33:19.464862818+01:00","with nospace":"with nospace"} -{"key":"\u001b[1G\u001b[K\u001b[1A","lvl":"info","msg":"Bash escapes in value","t":"2023-11-09T08:33:19.464876802+01:00"} -{"\u001b[1G\u001b[K\u001b[1A":"value","lvl":"info","msg":"Bash escapes in key","t":"2023-11-09T08:33:19.464885416+01:00"} -{"key":"value","lvl":"info","msg":"Bash escapes in message \u001b[1G\u001b[K\u001b[1A end","t":"2023-11-09T08:33:19.464906946+01:00"} -{"\u001b[35mColored\u001b[0m[":"\u001b[35mColored\u001b[0m[","lvl":"info","msg":"\u001b[35mColored\u001b[0m[","t":"2023-11-09T08:33:19.464921455+01:00"} -{"2562047h47m16.854s":"2562047h47m16.854s","lvl":"info","msg":"Custom Stringer value","t":"2023-11-09T08:33:19.464943893+01:00"} -{"key":"lazy value","lvl":"info","msg":"Lazy evaluation of value","t":"2023-11-09T08:33:19.465013552+01:00"} -{"lvl":"info","msg":"A message with wonky 💩 characters","t":"2023-11-09T08:33:19.465069437+01:00"} -{"lvl":"info","msg":"A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩","t":"2023-11-09T08:33:19.465083053+01:00"} -{"lvl":"info","msg":"A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above","t":"2023-11-09T08:33:19.465104289+01:00"} -{"false":"false","lvl":"info","msg":"boolean","t":"2023-11-09T08:33:19.465117185+01:00","true":"true"} -{"foo":"beta","lvl":"info","msg":"repeated-key 1","t":"2023-11-09T08:33:19.465143425+01:00"} -{"lvl":"info","msg":"repeated-key 2","t":"2023-11-09T08:33:19.465156323+01:00","xx":"longer"} -{"lvl":"info","msg":"log at level info","t":"2023-11-09T08:33:19.465193158+01:00"} -{"lvl":"warn","msg":"log at level warn","t":"2023-11-09T08:33:19.465228964+01:00"} -{"lvl":"eror","msg":"log at level error","t":"2023-11-09T08:33:19.465240352+01:00"} -{"a":"aligned left","bar":"short","lvl":"info","msg":"test","t":"2023-11-09T08:33:19.465247226+01:00"} -{"a":1,"bar":"a long message","lvl":"info","msg":"test","t":"2023-11-09T08:33:19.465269028+01:00"} -{"a":"aligned right","bar":"short","lvl":"info","msg":"test","t":"2023-11-09T08:33:19.465313611+01:00"} -{"lvl":"info","msg":"The following logs should align so that the key-fields make 5 columns","t":"2023-11-09T08:33:19.465328188+01:00"} -{"gas":1123123,"hash":"0x0000000000000000000000000000000000000000000000000000000000001234","lvl":"info","msg":"Inserted known block","number":1012,"other":"first","t":"2023-11-09T08:33:19.465350507+01:00","txs":200} -{"gas":1123,"hash":"0x0000000000000000000000000000000000000000000000000000000000001235","lvl":"info","msg":"Inserted new block","number":1,"other":"second","t":"2023-11-09T08:33:19.465387952+01:00","txs":2} -{"gas":1,"hash":"0x0000000000000000000000000000000000000000000000000000000000012322","lvl":"info","msg":"Inserted known block","number":99,"other":"third","t":"2023-11-09T08:33:19.465406687+01:00","txs":10} -{"gas":99,"hash":"0x0000000000000000000000000000000000000000000000000000000000001234","lvl":"warn","msg":"Inserted known block","number":1012,"other":"fourth","t":"2023-11-09T08:33:19.465433025+01:00","txs":200} -{"\u003cnil\u003e":"\u003cnil\u003e","lvl":"info","msg":"(*big.Int)(nil)","t":"2023-11-09T08:33:19.465450283+01:00"} -{"\u003cnil\u003e":"nil","lvl":"info","msg":"(*uint256.Int)(nil)","t":"2023-11-09T08:33:19.465472953+01:00"} -{"lvl":"info","msg":"(fmt.Stringer)(nil)","res":"\u003cnil\u003e","t":"2023-11-09T08:33:19.465538633+01:00"} -{"lvl":"info","msg":"nil-concrete-stringer","res":"nil","t":"2023-11-09T08:33:19.465552355+01:00"} -{"lvl":"info","msg":"error(nil) ","res":"\u003cnil\u003e","t":"2023-11-09T08:33:19.465601029+01:00"} -{"lvl":"info","msg":"nil-concrete-error","res":"","t":"2023-11-09T08:33:19.46561622+01:00"} -{"lvl":"info","msg":"nil-custom-struct","res":"\u003cnil\u003e","t":"2023-11-09T08:33:19.465638888+01:00"} -{"lvl":"info","msg":"raw nil","res":"\u003cnil\u003e","t":"2023-11-09T08:33:19.465673664+01:00"} -{"lvl":"info","msg":"(*uint64)(nil)","res":"\u003cnil\u003e","t":"2023-11-09T08:33:19.465700264+01:00"} -{"level":"level","lvl":"lvl","msg":"msg","t":"t","time":"time"} +{"t":"2023-11-22T15:42:00.407963+08:00","lvl":"info","msg":"big.Int","111,222,333,444,555,678,999":"111222333444555678999"} +{"t":"2023-11-22T15:42:00.408084+08:00","lvl":"info","msg":"-big.Int","-111,222,333,444,555,678,999":"-111222333444555678999"} +{"t":"2023-11-22T15:42:00.408092+08:00","lvl":"info","msg":"big.Int","11,122,233,344,455,567,899,900":"11122233344455567899900"} +{"t":"2023-11-22T15:42:00.408097+08:00","lvl":"info","msg":"-big.Int","-11,122,233,344,455,567,899,900":"-11122233344455567899900"} +{"t":"2023-11-22T15:42:00.408127+08:00","lvl":"info","msg":"uint256","111,222,333,444,555,678,999":"111222333444555678999"} +{"t":"2023-11-22T15:42:00.408133+08:00","lvl":"info","msg":"uint256","11,122,233,344,455,567,899,900":"11122233344455567899900"} +{"t":"2023-11-22T15:42:00.408137+08:00","lvl":"info","msg":"int64","1,000,000":1000000} +{"t":"2023-11-22T15:42:00.408145+08:00","lvl":"info","msg":"int64","-1,000,000":-1000000} +{"t":"2023-11-22T15:42:00.408149+08:00","lvl":"info","msg":"int64","9,223,372,036,854,775,807":9223372036854775807} +{"t":"2023-11-22T15:42:00.408153+08:00","lvl":"info","msg":"int64","-9,223,372,036,854,775,808":-9223372036854775808} +{"t":"2023-11-22T15:42:00.408156+08:00","lvl":"info","msg":"uint64","1,000,000":1000000} +{"t":"2023-11-22T15:42:00.40816+08:00","lvl":"info","msg":"uint64","18,446,744,073,709,551,615":18446744073709551615} +{"t":"2023-11-22T15:42:00.408164+08:00","lvl":"info","msg":"Special chars in value","key":"special \r\n\t chars"} +{"t":"2023-11-22T15:42:00.408167+08:00","lvl":"info","msg":"Special chars in key","special \n\t chars":"value"} +{"t":"2023-11-22T15:42:00.408171+08:00","lvl":"info","msg":"nospace","nospace":"nospace"} +{"t":"2023-11-22T15:42:00.408174+08:00","lvl":"info","msg":"with space","with nospace":"with nospace"} +{"t":"2023-11-22T15:42:00.408178+08:00","lvl":"info","msg":"Bash escapes in value","key":"\u001b[1G\u001b[K\u001b[1A"} +{"t":"2023-11-22T15:42:00.408182+08:00","lvl":"info","msg":"Bash escapes in key","\u001b[1G\u001b[K\u001b[1A":"value"} +{"t":"2023-11-22T15:42:00.408186+08:00","lvl":"info","msg":"Bash escapes in message \u001b[1G\u001b[K\u001b[1A end","key":"value"} +{"t":"2023-11-22T15:42:00.408194+08:00","lvl":"info","msg":"\u001b[35mColored\u001b[0m[","\u001b[35mColored\u001b[0m[":"\u001b[35mColored\u001b[0m["} +{"t":"2023-11-22T15:42:00.408197+08:00","lvl":"info","msg":"an error message with quotes","error":"this is an 'error'"} +{"t":"2023-11-22T15:42:00.408202+08:00","lvl":"info","msg":"Custom Stringer value","2562047h47m16.854s":"2562047h47m16.854s"} +{"t":"2023-11-22T15:42:00.408208+08:00","lvl":"info","msg":"a custom stringer that emits quoted text","output":"output with 'quotes'"} +{"t":"2023-11-22T15:42:00.408215+08:00","lvl":"info","msg":"Lazy evaluation of value","key":"lazy value"} +{"t":"2023-11-22T15:42:00.408219+08:00","lvl":"info","msg":"A message with wonky 💩 characters"} +{"t":"2023-11-22T15:42:00.408222+08:00","lvl":"info","msg":"A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩"} +{"t":"2023-11-22T15:42:00.408226+08:00","lvl":"info","msg":"A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above"} +{"t":"2023-11-22T15:42:00.408229+08:00","lvl":"info","msg":"boolean","true":true,"false":false} +{"t":"2023-11-22T15:42:00.408234+08:00","lvl":"info","msg":"repeated-key 1","foo":"alpha","foo":"beta"} +{"t":"2023-11-22T15:42:00.408237+08:00","lvl":"info","msg":"repeated-key 2","xx":"short","xx":"longer"} +{"t":"2023-11-22T15:42:00.408241+08:00","lvl":"info","msg":"log at level info"} +{"t":"2023-11-22T15:42:00.408244+08:00","lvl":"warn","msg":"log at level warn"} +{"t":"2023-11-22T15:42:00.408247+08:00","lvl":"eror","msg":"log at level error"} +{"t":"2023-11-22T15:42:00.408251+08:00","lvl":"info","msg":"test","bar":"short","a":"aligned left"} +{"t":"2023-11-22T15:42:00.408254+08:00","lvl":"info","msg":"test","bar":"a long message","a":1} +{"t":"2023-11-22T15:42:00.408258+08:00","lvl":"info","msg":"test","bar":"short","a":"aligned right"} +{"t":"2023-11-22T15:42:00.408261+08:00","lvl":"info","msg":"The following logs should align so that the key-fields make 5 columns"} +{"t":"2023-11-22T15:42:00.408275+08:00","lvl":"info","msg":"Inserted known block","number":1012,"hash":"0x0000000000000000000000000000000000000000000000000000000000001234","txs":200,"gas":1123123,"other":"first"} +{"t":"2023-11-22T15:42:00.408281+08:00","lvl":"info","msg":"Inserted new block","number":1,"hash":"0x0000000000000000000000000000000000000000000000000000000000001235","txs":2,"gas":1123,"other":"second"} +{"t":"2023-11-22T15:42:00.408287+08:00","lvl":"info","msg":"Inserted known block","number":99,"hash":"0x0000000000000000000000000000000000000000000000000000000000012322","txs":10,"gas":1,"other":"third"} +{"t":"2023-11-22T15:42:00.408296+08:00","lvl":"warn","msg":"Inserted known block","number":1012,"hash":"0x0000000000000000000000000000000000000000000000000000000000001234","txs":200,"gas":99,"other":"fourth"} +{"t":"2023-11-22T15:42:00.4083+08:00","lvl":"info","msg":"(*big.Int)(nil)","":""} +{"t":"2023-11-22T15:42:00.408303+08:00","lvl":"info","msg":"(*uint256.Int)(nil)","":""} +{"t":"2023-11-22T15:42:00.408311+08:00","lvl":"info","msg":"(fmt.Stringer)(nil)","res":null} +{"t":"2023-11-22T15:42:00.408318+08:00","lvl":"info","msg":"nil-concrete-stringer","res":""} +{"t":"2023-11-22T15:42:00.408322+08:00","lvl":"info","msg":"error(nil) ","res":null} +{"t":"2023-11-22T15:42:00.408326+08:00","lvl":"info","msg":"nil-concrete-error","res":""} +{"t":"2023-11-22T15:42:00.408334+08:00","lvl":"info","msg":"nil-custom-struct","res":null} +{"t":"2023-11-22T15:42:00.40835+08:00","lvl":"info","msg":"raw nil","res":null} +{"t":"2023-11-22T15:42:00.408354+08:00","lvl":"info","msg":"(*uint64)(nil)","res":null} +{"t":"2023-11-22T15:42:00.408361+08:00","lvl":"info","msg":"Using keys 't', 'lvl', 'time', 'level' and 'msg'","t":"t","time":"time","lvl":"lvl","level":"level","msg":"msg"} diff --git a/cmd/geth/testdata/logging/logtest-logfmt.txt b/cmd/geth/testdata/logging/logtest-logfmt.txt index f097143a55..114569e467 100644 --- a/cmd/geth/testdata/logging/logtest-logfmt.txt +++ b/cmd/geth/testdata/logging/logtest-logfmt.txt @@ -1,51 +1,51 @@ -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=-big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=big.Int 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=-big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=uint256 111,222,333,444,555,678,999=111,222,333,444,555,678,999 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=uint256 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=int64 1,000,000=1,000,000 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=int64 -1,000,000=-1,000,000 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=int64 9,223,372,036,854,775,807=9,223,372,036,854,775,807 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=int64 -9,223,372,036,854,775,808=-9,223,372,036,854,775,808 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=uint64 1,000,000=1,000,000 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=uint64 18,446,744,073,709,551,615=18,446,744,073,709,551,615 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Special chars in value" key="special \r\n\t chars" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Special chars in key" "special \n\t chars"=value -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=nospace nospace=nospace -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="with space" "with nospace"="with nospace" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Bash escapes in value" key="\x1b[1G\x1b[K\x1b[1A" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Bash escapes in key" "\x1b[1G\x1b[K\x1b[1A"=value -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="an error message with quotes" error="this is an 'error'" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Custom Stringer value" 2562047h47m16.854s=2562047h47m16.854s -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="a custom stringer that emits quoted text" output="output with 'quotes'" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Lazy evaluation of value" key="lazy value" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="A message with wonky 💩 characters" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=boolean true=true false=false -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="repeated-key 1" foo=alpha foo=beta -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="repeated-key 2" xx=short xx=longer -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="log at level info" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=warn msg="log at level warn" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=eror msg="log at level error" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=test bar=short a="aligned left" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=test bar="a long message" a=1 -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=test bar=short a="aligned right" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="The following logs should align so that the key-fields make 5 columns" -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=1,123,123 other=first -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Inserted new block" number=1 hash=0x0000000000000000000000000000000000000000000000000000000000001235 txs=2 gas=1123 other=second -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Inserted known block" number=99 hash=0x0000000000000000000000000000000000000000000000000000000000012322 txs=10 gas=1 other=third -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=warn msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=99 other=fourth -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=(*big.Int)(nil) = -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=(*uint256.Int)(nil) = -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=(fmt.Stringer)(nil) res=nil -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=nil-concrete-stringer res=nil -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="error(nil) " res=nil -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=nil-concrete-error res= -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=nil-custom-struct res= -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="raw nil" res=nil -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg=(*uint64)(nil) res= -t=xxxxxxxxxxxxxxxxxxxxxxxx lvl=info msg="Using keys 't', 'lvl', 'time', 'level' and 'msg'" t=t time=time lvl=lvl level=level msg=msg +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=big.Int 111,222,333,444,555,678,999=111222333444555678999 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=-big.Int -111,222,333,444,555,678,999=-111222333444555678999 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=big.Int 11,122,233,344,455,567,899,900=11122233344455567899900 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=-big.Int -11,122,233,344,455,567,899,900=-11122233344455567899900 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=uint256 111,222,333,444,555,678,999=111222333444555678999 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=uint256 11,122,233,344,455,567,899,900=11122233344455567899900 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=int64 1,000,000=1000000 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=int64 -1,000,000=-1000000 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=int64 9,223,372,036,854,775,807=9223372036854775807 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=int64 -9,223,372,036,854,775,808=-9223372036854775808 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=uint64 1,000,000=1000000 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=uint64 18,446,744,073,709,551,615=18446744073709551615 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Special chars in value" key="special \r\n\t chars" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Special chars in key" "special \n\t chars"=value +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nospace nospace=nospace +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="with space" "with nospace"="with nospace" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Bash escapes in value" key="\x1b[1G\x1b[K\x1b[1A" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Bash escapes in key" "\x1b[1G\x1b[K\x1b[1A"=value +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="an error message with quotes" error="this is an 'error'" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Custom Stringer value" 2562047h47m16.854s=2562047h47m16.854s +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="a custom stringer that emits quoted text" output="output with 'quotes'" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Lazy evaluation of value" key="lazy value" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A message with wonky 💩 characters" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=boolean true=true false=false +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="repeated-key 1" foo=alpha foo=beta +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="repeated-key 2" xx=short xx=longer +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="log at level info" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=warn msg="log at level warn" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=eror msg="log at level error" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar=short a="aligned left" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar="a long message" a=1 +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=test bar=short a="aligned right" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="The following logs should align so that the key-fields make 5 columns" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=1123123 other=first +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Inserted new block" number=1 hash=0x0000000000000000000000000000000000000000000000000000000000001235 txs=2 gas=1123 other=second +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Inserted known block" number=99 hash=0x0000000000000000000000000000000000000000000000000000000000012322 txs=10 gas=1 other=third +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=warn msg="Inserted known block" number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=99 other=fourth +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(*big.Int)(nil) = +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(*uint256.Int)(nil) = +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(fmt.Stringer)(nil) res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nil-concrete-stringer res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="error(nil) " res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nil-concrete-error res="" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nil-custom-struct res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="raw nil" res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(*uint64)(nil) res= +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Using keys 't', 'lvl', 'time', 'level' and 'msg'" t=t time=time lvl=lvl level=level msg=msg diff --git a/cmd/geth/testdata/logging/logtest-terminal.txt b/cmd/geth/testdata/logging/logtest-terminal.txt index 051a6267fa..4da3f49d46 100644 --- a/cmd/geth/testdata/logging/logtest-terminal.txt +++ b/cmd/geth/testdata/logging/logtest-terminal.txt @@ -1,52 +1,52 @@ -INFO [XX-XX|XX:XX:XX.XXX] big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999 -INFO [XX-XX|XX:XX:XX.XXX] -big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999 -INFO [XX-XX|XX:XX:XX.XXX] big.Int 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 -INFO [XX-XX|XX:XX:XX.XXX] -big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900 -INFO [XX-XX|XX:XX:XX.XXX] uint256 111,222,333,444,555,678,999=111,222,333,444,555,678,999 -INFO [XX-XX|XX:XX:XX.XXX] uint256 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 -INFO [XX-XX|XX:XX:XX.XXX] int64 1,000,000=1,000,000 -INFO [XX-XX|XX:XX:XX.XXX] int64 -1,000,000=-1,000,000 -INFO [XX-XX|XX:XX:XX.XXX] int64 9,223,372,036,854,775,807=9,223,372,036,854,775,807 -INFO [XX-XX|XX:XX:XX.XXX] int64 -9,223,372,036,854,775,808=-9,223,372,036,854,775,808 -INFO [XX-XX|XX:XX:XX.XXX] uint64 1,000,000=1,000,000 -INFO [XX-XX|XX:XX:XX.XXX] uint64 18,446,744,073,709,551,615=18,446,744,073,709,551,615 -INFO [XX-XX|XX:XX:XX.XXX] Special chars in value key="special \r\n\t chars" -INFO [XX-XX|XX:XX:XX.XXX] Special chars in key "special \n\t chars"=value -INFO [XX-XX|XX:XX:XX.XXX] nospace nospace=nospace -INFO [XX-XX|XX:XX:XX.XXX] with space "with nospace"="with nospace" -INFO [XX-XX|XX:XX:XX.XXX] Bash escapes in value key="\x1b[1G\x1b[K\x1b[1A" -INFO [XX-XX|XX:XX:XX.XXX] Bash escapes in key "\x1b[1G\x1b[K\x1b[1A"=value -INFO [XX-XX|XX:XX:XX.XXX] "Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value -INFO [XX-XX|XX:XX:XX.XXX] "\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" -INFO [XX-XX|XX:XX:XX.XXX] an error message with quotes error="this is an 'error'" -INFO [XX-XX|XX:XX:XX.XXX] Custom Stringer value 2562047h47m16.854s=2562047h47m16.854s -INFO [XX-XX|XX:XX:XX.XXX] a custom stringer that emits quoted text output="output with 'quotes'" -INFO [XX-XX|XX:XX:XX.XXX] Lazy evaluation of value key="lazy value" -INFO [XX-XX|XX:XX:XX.XXX] "A message with wonky 💩 characters" -INFO [XX-XX|XX:XX:XX.XXX] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" -INFO [XX-XX|XX:XX:XX.XXX] A multiline message +INFO [xx-xx|xx:xx:xx.xxx] big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +INFO [xx-xx|xx:xx:xx.xxx] -big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999 +INFO [xx-xx|xx:xx:xx.xxx] big.Int 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +INFO [xx-xx|xx:xx:xx.xxx] -big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900 +INFO [xx-xx|xx:xx:xx.xxx] uint256 111,222,333,444,555,678,999=111,222,333,444,555,678,999 +INFO [xx-xx|xx:xx:xx.xxx] uint256 11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900 +INFO [xx-xx|xx:xx:xx.xxx] int64 1,000,000=1,000,000 +INFO [xx-xx|xx:xx:xx.xxx] int64 -1,000,000=-1,000,000 +INFO [xx-xx|xx:xx:xx.xxx] int64 9,223,372,036,854,775,807=9,223,372,036,854,775,807 +INFO [xx-xx|xx:xx:xx.xxx] int64 -9,223,372,036,854,775,808=-9,223,372,036,854,775,808 +INFO [xx-xx|xx:xx:xx.xxx] uint64 1,000,000=1,000,000 +INFO [xx-xx|xx:xx:xx.xxx] uint64 18,446,744,073,709,551,615=18,446,744,073,709,551,615 +INFO [xx-xx|xx:xx:xx.xxx] Special chars in value key="special \r\n\t chars" +INFO [xx-xx|xx:xx:xx.xxx] Special chars in key "special \n\t chars"=value +INFO [xx-xx|xx:xx:xx.xxx] nospace nospace=nospace +INFO [xx-xx|xx:xx:xx.xxx] with space "with nospace"="with nospace" +INFO [xx-xx|xx:xx:xx.xxx] Bash escapes in value key="\x1b[1G\x1b[K\x1b[1A" +INFO [xx-xx|xx:xx:xx.xxx] Bash escapes in key "\x1b[1G\x1b[K\x1b[1A"=value +INFO [xx-xx|xx:xx:xx.xxx] "Bash escapes in message \x1b[1G\x1b[K\x1b[1A end" key=value +INFO [xx-xx|xx:xx:xx.xxx] "\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m[" +INFO [xx-xx|xx:xx:xx.xxx] an error message with quotes error="this is an 'error'" +INFO [xx-xx|xx:xx:xx.xxx] Custom Stringer value 2562047h47m16.854s=2562047h47m16.854s +INFO [xx-xx|xx:xx:xx.xxx] a custom stringer that emits quoted text output="output with 'quotes'" +INFO [xx-xx|xx:xx:xx.xxx] Lazy evaluation of value key="lazy value" +INFO [xx-xx|xx:xx:xx.xxx] "A message with wonky 💩 characters" +INFO [xx-xx|xx:xx:xx.xxx] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" +INFO [xx-xx|xx:xx:xx.xxx] A multiline message LALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above -INFO [XX-XX|XX:XX:XX.XXX] boolean true=true false=false -INFO [XX-XX|XX:XX:XX.XXX] repeated-key 1 foo=alpha foo=beta -INFO [XX-XX|XX:XX:XX.XXX] repeated-key 2 xx=short xx=longer -INFO [XX-XX|XX:XX:XX.XXX] log at level info -WARN [XX-XX|XX:XX:XX.XXX] log at level warn -ERROR[XX-XX|XX:XX:XX.XXX] log at level error -INFO [XX-XX|XX:XX:XX.XXX] test bar=short a="aligned left" -INFO [XX-XX|XX:XX:XX.XXX] test bar="a long message" a=1 -INFO [XX-XX|XX:XX:XX.XXX] test bar=short a="aligned right" -INFO [XX-XX|XX:XX:XX.XXX] The following logs should align so that the key-fields make 5 columns -INFO [XX-XX|XX:XX:XX.XXX] Inserted known block number=1012 hash=000000..001234 txs=200 gas=1,123,123 other=first -INFO [XX-XX|XX:XX:XX.XXX] Inserted new block number=1 hash=000000..001235 txs=2 gas=1123 other=second -INFO [XX-XX|XX:XX:XX.XXX] Inserted known block number=99 hash=000000..012322 txs=10 gas=1 other=third -WARN [XX-XX|XX:XX:XX.XXX] Inserted known block number=1012 hash=000000..001234 txs=200 gas=99 other=fourth -INFO [XX-XX|XX:XX:XX.XXX] (*big.Int)(nil) = -INFO [XX-XX|XX:XX:XX.XXX] (*uint256.Int)(nil) = -INFO [XX-XX|XX:XX:XX.XXX] (fmt.Stringer)(nil) res=nil -INFO [XX-XX|XX:XX:XX.XXX] nil-concrete-stringer res=nil -INFO [XX-XX|XX:XX:XX.XXX] error(nil) res=nil -INFO [XX-XX|XX:XX:XX.XXX] nil-concrete-error res= -INFO [XX-XX|XX:XX:XX.XXX] nil-custom-struct res= -INFO [XX-XX|XX:XX:XX.XXX] raw nil res=nil -INFO [XX-XX|XX:XX:XX.XXX] (*uint64)(nil) res= -INFO [XX-XX|XX:XX:XX.XXX] Using keys 't', 'lvl', 'time', 'level' and 'msg' t=t time=time lvl=lvl level=level msg=msg +INFO [xx-xx|xx:xx:xx.xxx] boolean true=true false=false +INFO [xx-xx|xx:xx:xx.xxx] repeated-key 1 foo=alpha foo=beta +INFO [xx-xx|xx:xx:xx.xxx] repeated-key 2 xx=short xx=longer +INFO [xx-xx|xx:xx:xx.xxx] log at level info +WARN [xx-xx|xx:xx:xx.xxx] log at level warn +ERROR[xx-xx|xx:xx:xx.xxx] log at level error +INFO [xx-xx|xx:xx:xx.xxx] test bar=short a="aligned left" +INFO [xx-xx|xx:xx:xx.xxx] test bar="a long message" a=1 +INFO [xx-xx|xx:xx:xx.xxx] test bar=short a="aligned right" +INFO [xx-xx|xx:xx:xx.xxx] The following logs should align so that the key-fields make 5 columns +INFO [xx-xx|xx:xx:xx.xxx] Inserted known block number=1012 hash=000000..001234 txs=200 gas=1,123,123 other=first +INFO [xx-xx|xx:xx:xx.xxx] Inserted new block number=1 hash=000000..001235 txs=2 gas=1123 other=second +INFO [xx-xx|xx:xx:xx.xxx] Inserted known block number=99 hash=000000..012322 txs=10 gas=1 other=third +WARN [xx-xx|xx:xx:xx.xxx] Inserted known block number=1012 hash=000000..001234 txs=200 gas=99 other=fourth +INFO [xx-xx|xx:xx:xx.xxx] (*big.Int)(nil) = +INFO [xx-xx|xx:xx:xx.xxx] (*uint256.Int)(nil) = +INFO [xx-xx|xx:xx:xx.xxx] (fmt.Stringer)(nil) res= +INFO [xx-xx|xx:xx:xx.xxx] nil-concrete-stringer res= +INFO [xx-xx|xx:xx:xx.xxx] error(nil) res= +INFO [xx-xx|xx:xx:xx.xxx] nil-concrete-error res= +INFO [xx-xx|xx:xx:xx.xxx] nil-custom-struct res= +INFO [xx-xx|xx:xx:xx.xxx] raw nil res= +INFO [xx-xx|xx:xx:xx.xxx] (*uint64)(nil) res= +INFO [xx-xx|xx:xx:xx.xxx] Using keys 't', 'lvl', 'time', 'level' and 'msg' t=t time=time lvl=lvl level=level msg=msg diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b49c7c36d5..27e1b3f623 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1389,6 +1389,13 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { log.Info(fmt.Sprintf("Using %s as db engine", dbEngine)) cfg.DBEngine = dbEngine } + // deprecation notice for log debug flags (TODO: find a more appropriate place to put these?) + if ctx.IsSet(LogBacktraceAtFlag.Name) { + log.Warn("log.backtrace flag is deprecated") + } + if ctx.IsSet(LogDebugFlag.Name) { + log.Warn("log.debug flag is deprecated") + } } func setSmartCard(ctx *cli.Context, cfg *node.Config) { diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 00237fecaf..243abd8311 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -45,6 +45,8 @@ var DeprecatedFlags = []cli.Flag{ LightMaxPeersFlag, LightNoPruneFlag, LightNoSyncServeFlag, + LogBacktraceAtFlag, + LogDebugFlag, } var ( @@ -118,6 +120,18 @@ var ( Usage: "Enables serving light clients before syncing (deprecated)", Category: flags.LightCategory, } + // Deprecated November 2023 + LogBacktraceAtFlag = &cli.StringFlag{ + Name: "log.backtrace", + Usage: "Request a stack trace at a specific logging statement (deprecated)", + Value: "", + Category: flags.DeprecatedCategory, + } + LogDebugFlag = &cli.BoolFlag{ + Name: "log.debug", + Usage: "Prepends log messages with call-site location (deprecated)", + Category: flags.DeprecatedCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 07016b675c..c25f3e7e8b 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -601,7 +601,7 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) { } func enableLogging() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) } // Tests that snapshot generation when an extra account with storage exists in the snap state. diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 8914301e14..fa3e8edc90 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -319,7 +319,7 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { // - 3. All transactions after a nonce gap must be dropped // - 4. All transactions after an underpriced one (including it) must be dropped func TestOpenDrops(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) // Create a temporary folder for the persistent backend storage, _ := os.MkdirTemp("", "blobpool-") @@ -600,7 +600,7 @@ func TestOpenDrops(t *testing.T) { // - 2. Eviction thresholds are calculated correctly for the sequences // - 3. Balance usage of an account is totals across all transactions func TestOpenIndex(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) // Create a temporary folder for the persistent backend storage, _ := os.MkdirTemp("", "blobpool-") @@ -689,7 +689,7 @@ func TestOpenIndex(t *testing.T) { // Tests that after indexing all the loaded transactions from disk, a price heap // is correctly constructed based on the head basefee and blobfee. func TestOpenHeap(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) // Create a temporary folder for the persistent backend storage, _ := os.MkdirTemp("", "blobpool-") @@ -776,7 +776,7 @@ func TestOpenHeap(t *testing.T) { // Tests that after the pool's previous state is loaded back, any transactions // over the new storage cap will get dropped. func TestOpenCap(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) // Create a temporary folder for the persistent backend storage, _ := os.MkdirTemp("", "blobpool-") @@ -868,7 +868,7 @@ func TestOpenCap(t *testing.T) { // specific to the blob pool. It does not do an exhaustive transaction validity // check. func TestAdd(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) // seed is a helper tumpe to seed an initial state db and pool type seed struct { diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 59f44fafea..c875c485dd 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -1562,7 +1562,7 @@ func TestBlockToPayloadWithBlobs(t *testing.T) { // This checks that beaconRoot is applied to the state from the engine API. func TestParentBeaconBlockRoot(t *testing.T) { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(colorable.NewColorableStderr(), log.LevelTrace, true))) genesis, blocks := generateMergeChain(10, true) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index a8b1b45e00..50b9031a27 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/big" "math/rand" + "os" "sync" "testing" "time" @@ -31,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/exp/slog" ) // makeChain creates a chain of n blocks starting at and including parent. @@ -271,7 +273,7 @@ func XTestDelivery(t *testing.T) { world.chain = blo world.progress(10) if false { - log.Root().SetHandler(log.StdoutHandler) + log.SetDefault(log.NewLogger(slog.NewTextHandler(os.Stdout, nil))) } q := newQueue(10, 10) var wg sync.WaitGroup diff --git a/go.mod b/go.mod index 042d7155be..75c2b899d6 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 - github.com/go-stack/stack v1.8.1 github.com/gofrs/flock v0.8.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/golang/protobuf v1.5.3 @@ -64,13 +63,13 @@ require ( github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/sync v0.4.0 - golang.org/x/sys v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/crypto v0.15.0 + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.14.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.13.0 + golang.org/x/tools v0.15.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -136,8 +135,8 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.18.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/go.sum b/go.sum index d5e7901e5a..f89adbe571 100644 --- a/go.sum +++ b/go.sum @@ -228,8 +228,6 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= @@ -616,8 +614,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -628,8 +626,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -651,8 +649,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -692,8 +690,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -712,8 +710,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -776,8 +774,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -790,8 +788,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -846,8 +844,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.15.0 h1:zdAyfUGbYmuVokhzVmghFl2ZJh5QhcfebBgmVPFYA+8= +golang.org/x/tools v0.15.0/go.mod h1:hpksKq4dtpQWS1uQ61JkdqWM3LscIS6Slf+VVkm+wQk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/debug/api.go b/internal/debug/api.go index 42d0fa15ed..482989e0d0 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/hashicorp/go-bexpr" + "golang.org/x/exp/slog" ) // Handler is the global debugging handler. @@ -56,7 +57,7 @@ type HandlerT struct { // Verbosity sets the log verbosity ceiling. The verbosity of individual packages // and source files can be raised using Vmodule. func (*HandlerT) Verbosity(level int) { - glogger.Verbosity(log.Lvl(level)) + glogger.Verbosity(slog.Level(level)) } // Vmodule sets the log verbosity pattern. See package log for details on the @@ -65,12 +66,6 @@ func (*HandlerT) Vmodule(pattern string) error { return glogger.Vmodule(pattern) } -// BacktraceAt sets the log backtrace location. See package log for details on -// the pattern syntax. -func (*HandlerT) BacktraceAt(location string) error { - return glogger.BacktraceAt(location) -} - // MemStats returns detailed runtime memory statistics. func (*HandlerT) MemStats() *runtime.MemStats { s := new(runtime.MemStats) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 4f0f5fe860..23e4745e8c 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -34,6 +34,7 @@ import ( "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" + "golang.org/x/exp/slog" "gopkg.in/natefinch/lumberjack.v2" ) @@ -75,17 +76,6 @@ var ( Usage: "Write logs to a file", Category: flags.LoggingCategory, } - backtraceAtFlag = &cli.StringFlag{ - Name: "log.backtrace", - Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\")", - Value: "", - Category: flags.LoggingCategory, - } - debugFlag = &cli.BoolFlag{ - Name: "log.debug", - Usage: "Prepends log messages with call-site location (file and line number)", - Category: flags.LoggingCategory, - } logRotateFlag = &cli.BoolFlag{ Name: "log.rotate", Usage: "Enables log file rotation", @@ -160,8 +150,6 @@ var Flags = []cli.Flag{ verbosityFlag, logVmoduleFlag, vmoduleFlag, - backtraceAtFlag, - debugFlag, logjsonFlag, logFormatFlag, logFileFlag, @@ -180,45 +168,34 @@ var Flags = []cli.Flag{ } var ( - glogger *log.GlogHandler - logOutputStream log.Handler + glogger *log.GlogHandler + logOutputFile io.WriteCloser + defaultTerminalHandler *log.TerminalHandler ) func init() { - glogger = log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + defaultTerminalHandler = log.NewTerminalHandler(os.Stderr, false) + glogger = log.NewGlogHandler(defaultTerminalHandler) glogger.Verbosity(log.LvlInfo) - log.Root().SetHandler(glogger) + log.SetDefault(log.NewLogger(glogger)) +} + +func ResetLogging() { + if defaultTerminalHandler != nil { + defaultTerminalHandler.ResetFieldPadding() + } } // Setup initializes profiling and logging based on the CLI flags. // It should be called as early as possible in the program. func Setup(ctx *cli.Context) error { var ( - logfmt log.Format - output = io.Writer(os.Stderr) - logFmtFlag = ctx.String(logFormatFlag.Name) + handler slog.Handler + terminalOutput = io.Writer(os.Stderr) + output io.Writer + logFmtFlag = ctx.String(logFormatFlag.Name) ) - switch { - case ctx.Bool(logjsonFlag.Name): - // Retain backwards compatibility with `--log.json` flag if `--log.format` not set - defer log.Warn("The flag '--log.json' is deprecated, please use '--log.format=json' instead") - logfmt = log.JSONFormat() - case logFmtFlag == "json": - logfmt = log.JSONFormat() - case logFmtFlag == "logfmt": - logfmt = log.LogfmtFormat() - case logFmtFlag == "", logFmtFlag == "terminal": - useColor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" - if useColor { - output = colorable.NewColorableStderr() - } - logfmt = log.TerminalFormat(useColor) - default: - // Unknown log format specified - return fmt.Errorf("unknown log format: %v", ctx.String(logFormatFlag.Name)) - } var ( - ostream = log.StreamHandler(output, logfmt) logFile = ctx.String(logFileFlag.Name) rotation = ctx.Bool(logRotateFlag.Name) ) @@ -241,27 +218,55 @@ func Setup(ctx *cli.Context) error { } else { context = append(context, "location", filepath.Join(os.TempDir(), "geth-lumberjack.log")) } - lumberWriter := &lumberjack.Logger{ + logOutputFile = &lumberjack.Logger{ Filename: logFile, MaxSize: ctx.Int(logMaxSizeMBsFlag.Name), MaxBackups: ctx.Int(logMaxBackupsFlag.Name), MaxAge: ctx.Int(logMaxAgeFlag.Name), Compress: ctx.Bool(logCompressFlag.Name), } - ostream = log.StreamHandler(io.MultiWriter(output, lumberWriter), logfmt) + output = io.MultiWriter(terminalOutput, logOutputFile) } else if logFile != "" { - f, err := os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { + var err error + if logOutputFile, err = os.OpenFile(logFile, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { return err } - ostream = log.StreamHandler(io.MultiWriter(output, f), logfmt) + output = io.MultiWriter(logOutputFile, terminalOutput) context = append(context, "location", logFile) + } else { + output = terminalOutput } - glogger.SetHandler(ostream) + + switch { + case ctx.Bool(logjsonFlag.Name): + // Retain backwards compatibility with `--log.json` flag if `--log.format` not set + defer log.Warn("The flag '--log.json' is deprecated, please use '--log.format=json' instead") + handler = log.JSONHandler(output) + case logFmtFlag == "json": + handler = log.JSONHandler(output) + case logFmtFlag == "logfmt": + handler = log.LogfmtHandler(output) + case logFmtFlag == "", logFmtFlag == "terminal": + useColor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" + if useColor { + terminalOutput = colorable.NewColorableStderr() + if logOutputFile != nil { + output = io.MultiWriter(logOutputFile, terminalOutput) + } else { + output = terminalOutput + } + } + handler = log.NewTerminalHandler(output, useColor) + default: + // Unknown log format specified + return fmt.Errorf("unknown log format: %v", ctx.String(logFormatFlag.Name)) + } + + glogger = log.NewGlogHandler(handler) // logging - verbosity := ctx.Int(verbosityFlag.Name) - glogger.Verbosity(log.Lvl(verbosity)) + verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name)) + glogger.Verbosity(verbosity) vmodule := ctx.String(logVmoduleFlag.Name) if vmodule == "" { // Retain backwards compatibility with `--vmodule` flag if `--log.vmodule` not set @@ -272,16 +277,7 @@ func Setup(ctx *cli.Context) error { } glogger.Vmodule(vmodule) - debug := ctx.Bool(debugFlag.Name) - if ctx.IsSet(debugFlag.Name) { - debug = ctx.Bool(debugFlag.Name) - } - log.PrintOrigins(debug) - - backtrace := ctx.String(backtraceAtFlag.Name) - glogger.BacktraceAt(backtrace) - - log.Root().SetHandler(glogger) + log.SetDefault(log.NewLogger(glogger)) // profiling, tracing runtime.MemProfileRate = memprofilerateFlag.Value @@ -341,8 +337,8 @@ func StartPProf(address string, withMetrics bool) { func Exit() { Handler.StopCPUProfile() Handler.StopGoTrace() - if closer, ok := logOutputStream.(io.Closer); ok { - closer.Close() + if logOutputFile != nil { + logOutputFile.Close() } } diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index 684339f16d..68b9fb19f8 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -18,26 +18,19 @@ package testlog import ( + "bytes" + "context" + "fmt" "sync" "testing" "github.com/ethereum/go-ethereum/log" + "golang.org/x/exp/slog" ) -// Handler returns a log handler which logs to the unit test log of t. -func Handler(t *testing.T, level log.Lvl) log.Handler { - return log.LvlFilterHandler(level, &handler{t, log.TerminalFormat(false)}) -} - -type handler struct { - t *testing.T - fmt log.Format -} - -func (h *handler) Log(r *log.Record) error { - h.t.Logf("%s", h.fmt.Format(r)) - return nil -} +const ( + termTimeFormat = "01-02|15:04:05.000" +) // logger implements log.Logger such that all output goes to the unit test log via // t.Logf(). All methods in between logger.Trace, logger.Debug, etc. are marked as test @@ -51,27 +44,62 @@ type logger struct { } type bufHandler struct { - buf []*log.Record - fmt log.Format + buf []slog.Record + attrs []slog.Attr + level slog.Level } -func (h *bufHandler) Log(r *log.Record) error { +func (h *bufHandler) Handle(_ context.Context, r slog.Record) error { h.buf = append(h.buf, r) return nil } -// Logger returns a logger which logs to the unit test log of t. -func Logger(t *testing.T, level log.Lvl) log.Logger { - l := &logger{ - t: t, - l: log.New(), - mu: new(sync.Mutex), - h: &bufHandler{fmt: log.TerminalFormat(false)}, - } - l.l.SetHandler(log.LvlFilterHandler(level, l.h)) - return l +func (h *bufHandler) Enabled(_ context.Context, lvl slog.Level) bool { + return lvl <= h.level } +func (h *bufHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + records := make([]slog.Record, len(h.buf)) + copy(records[:], h.buf[:]) + return &bufHandler{ + records, + append(h.attrs, attrs...), + h.level, + } +} + +func (h *bufHandler) WithGroup(_ string) slog.Handler { + panic("not implemented") +} + +// Logger returns a logger which logs to the unit test log of t. +func Logger(t *testing.T, level slog.Level) log.Logger { + handler := bufHandler{ + []slog.Record{}, + []slog.Attr{}, + level, + } + return &logger{ + t: t, + l: log.NewLogger(&handler), + mu: new(sync.Mutex), + h: &handler, + } +} + +// LoggerWithHandler returns +func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { + var bh bufHandler + return &logger{ + t: t, + l: log.NewLogger(handler), + mu: new(sync.Mutex), + h: &bh, + } +} + +func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} + func (l *logger) Trace(msg string, ctx ...interface{}) { l.t.Helper() l.mu.Lock() @@ -80,6 +108,14 @@ func (l *logger) Trace(msg string, ctx ...interface{}) { l.flush() } +func (l *logger) Log(level slog.Level, msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Log(level, msg, ctx...) + l.flush() +} + func (l *logger) Debug(msg string, ctx ...interface{}) { l.t.Helper() l.mu.Lock() @@ -120,23 +156,45 @@ func (l *logger) Crit(msg string, ctx ...interface{}) { l.flush() } +func (l *logger) With(ctx ...interface{}) log.Logger { + return &logger{l.t, l.l.With(ctx...), l.mu, l.h} +} + func (l *logger) New(ctx ...interface{}) log.Logger { - return &logger{l.t, l.l.New(ctx...), l.mu, l.h} + return l.With(ctx...) } -func (l *logger) GetHandler() log.Handler { - return l.l.GetHandler() -} +// terminalFormat formats a message similarly to the NewTerminalHandler in the log package. +// The difference is that terminalFormat does not escape messages/attributes and does not pad attributes. +func (h *bufHandler) terminalFormat(r slog.Record) string { + buf := &bytes.Buffer{} + lvl := log.LevelAlignedString(r.Level) + attrs := []slog.Attr{} + r.Attrs(func(attr slog.Attr) bool { + attrs = append(attrs, attr) + return true + }) -func (l *logger) SetHandler(h log.Handler) { - l.l.SetHandler(h) + attrs = append(h.attrs, attrs...) + + fmt.Fprintf(buf, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), r.Message) + if length := len(r.Message); length < 40 { + buf.Write(bytes.Repeat([]byte{' '}, 40-length)) + } + + for _, attr := range attrs { + rawVal := attr.Value.Any() + fmt.Fprintf(buf, " %s=%s", attr.Key, log.FormatLogfmtValue(rawVal, true)) + } + buf.WriteByte('\n') + return buf.String() } // flush writes all buffered messages and clears the buffer. func (l *logger) flush() { l.t.Helper() for _, r := range l.h.buf { - l.t.Logf("%s", l.h.fmt.Format(r)) + l.t.Logf("%s", l.h.terminalFormat(r)) } l.h.buf = nil } diff --git a/log/CONTRIBUTORS b/log/CONTRIBUTORS deleted file mode 100644 index a0866713be..0000000000 --- a/log/CONTRIBUTORS +++ /dev/null @@ -1,11 +0,0 @@ -Contributors to log15: - -- Aaron L -- Alan Shreve -- Chris Hines -- Ciaran Downey -- Dmitry Chestnykh -- Evan Shaw -- Péter Szilágyi -- Trevor Gattis -- Vincent Vanackere diff --git a/log/LICENSE b/log/LICENSE deleted file mode 100644 index 5f0d1fb6a7..0000000000 --- a/log/LICENSE +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2014 Alan Shreve - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/log/README.md b/log/README.md deleted file mode 100644 index 47426806dd..0000000000 --- a/log/README.md +++ /dev/null @@ -1,77 +0,0 @@ -![obligatory xkcd](https://imgs.xkcd.com/comics/standards.png) - -# log15 [![godoc reference](https://godoc.org/github.com/inconshreveable/log15?status.png)](https://godoc.org/github.com/inconshreveable/log15) [![Build Status](https://travis-ci.org/inconshreveable/log15.svg?branch=master)](https://travis-ci.org/inconshreveable/log15) - -Package log15 provides an opinionated, simple toolkit for best-practice logging in Go (golang) that is both human and machine readable. It is modeled after the Go standard library's [`io`](https://golang.org/pkg/io/) and [`net/http`](https://golang.org/pkg/net/http/) packages and is an alternative to the standard library's [`log`](https://golang.org/pkg/log/) package. - -## Features -- A simple, easy-to-understand API -- Promotes structured logging by encouraging use of key/value pairs -- Child loggers which inherit and add their own private context -- Lazy evaluation of expensive operations -- Simple Handler interface allowing for construction of flexible, custom logging configurations with a tiny API. -- Color terminal support -- Built-in support for logging to files, streams, syslog, and the network -- Support for forking records to multiple handlers, buffering records for output, failing over from failed handler writes, + more - -## Versioning -The API of the master branch of log15 should always be considered unstable. If you want to rely on a stable API, -you must vendor the library. - -## Importing - -```go -import log "github.com/inconshreveable/log15" -``` - -## Examples - -```go -// all loggers can have key/value context -srvlog := log.New("module", "app/server") - -// all log messages can have key/value context -srvlog.Warn("abnormal conn rate", "rate", curRate, "low", lowRate, "high", highRate) - -// child loggers with inherited context -connlog := srvlog.New("raddr", c.RemoteAddr()) -connlog.Info("connection open") - -// lazy evaluation -connlog.Debug("ping remote", "latency", log.Lazy{pingRemote}) - -// flexible configuration -srvlog.SetHandler(log.MultiHandler( - log.StreamHandler(os.Stderr, log.LogfmtFormat()), - log.LvlFilterHandler( - log.LvlError, - log.Must.FileHandler("errors.json", log.JSONFormat())))) -``` - -Will result in output that looks like this: - -``` -WARN[06-17|21:58:10] abnormal conn rate module=app/server rate=0.500 low=0.100 high=0.800 -INFO[06-17|21:58:10] connection open module=app/server raddr=10.0.0.1 -``` - -## Breaking API Changes -The following commits broke API stability. This reference is intended to help you understand the consequences of updating to a newer version -of log15. - -- 57a084d014d4150152b19e4e531399a7145d1540 - Added a `Get()` method to the `Logger` interface to retrieve the current handler -- 93404652ee366648fa622b64d1e2b67d75a3094a - `Record` field `Call` changed to `stack.Call` with switch to `github.com/go-stack/stack` -- a5e7613673c73281f58e15a87d2cf0cf111e8152 - Restored `syslog.Priority` argument to the `SyslogXxx` handler constructors - -## FAQ - -### The varargs style is brittle and error prone! Can I have type safety please? -Yes. Use `log.Ctx`: - -```go -srvlog := log.New(log.Ctx{"module": "app/server"}) -srvlog.Warn("abnormal conn rate", log.Ctx{"rate": curRate, "low": lowRate, "high": highRate}) -``` - -## License -Apache diff --git a/log/README_ETHEREUM.md b/log/README_ETHEREUM.md deleted file mode 100644 index f6c42ccc03..0000000000 --- a/log/README_ETHEREUM.md +++ /dev/null @@ -1,5 +0,0 @@ -This package is a fork of https://github.com/inconshreveable/log15, with some -minor modifications required by the go-ethereum codebase: - - * Support for log level `trace` - * Modified behavior to exit on `critical` failure diff --git a/log/doc.go b/log/doc.go deleted file mode 100644 index d2e15140e4..0000000000 --- a/log/doc.go +++ /dev/null @@ -1,327 +0,0 @@ -/* -Package log15 provides an opinionated, simple toolkit for best-practice logging that is -both human and machine readable. It is modeled after the standard library's io and net/http -packages. - -This package enforces you to only log key/value pairs. Keys must be strings. Values may be -any type that you like. The default output format is logfmt, but you may also choose to use -JSON instead if that suits you. Here's how you log: - - log.Info("page accessed", "path", r.URL.Path, "user_id", user.id) - -This will output a line that looks like: - - lvl=info t=2014-05-02T16:07:23-0700 msg="page accessed" path=/org/71/profile user_id=9 - -# Getting Started - -To get started, you'll want to import the library: - - import log "github.com/inconshreveable/log15" - -Now you're ready to start logging: - - func main() { - log.Info("Program starting", "args", os.Args()) - } - -# Convention - -Because recording a human-meaningful message is common and good practice, the first argument to every -logging method is the value to the *implicit* key 'msg'. - -Additionally, the level you choose for a message will be automatically added with the key 'lvl', and so -will the current timestamp with key 't'. - -You may supply any additional context as a set of key/value pairs to the logging function. log15 allows -you to favor terseness, ordering, and speed over safety. This is a reasonable tradeoff for -logging functions. You don't need to explicitly state keys/values, log15 understands that they alternate -in the variadic argument list: - - log.Warn("size out of bounds", "low", lowBound, "high", highBound, "val", val) - -If you really do favor your type-safety, you may choose to pass a log.Ctx instead: - - log.Warn("size out of bounds", log.Ctx{"low": lowBound, "high": highBound, "val": val}) - -# Context loggers - -Frequently, you want to add context to a logger so that you can track actions associated with it. An http -request is a good example. You can easily create new loggers that have context that is automatically included -with each log line: - - requestlogger := log.New("path", r.URL.Path) - - // later - requestlogger.Debug("db txn commit", "duration", txnTimer.Finish()) - -This will output a log line that includes the path context that is attached to the logger: - - lvl=dbug t=2014-05-02T16:07:23-0700 path=/repo/12/add_hook msg="db txn commit" duration=0.12 - -# Handlers - -The Handler interface defines where log lines are printed to and how they are formatted. Handler is a -single interface that is inspired by net/http's handler interface: - - type Handler interface { - Log(r *Record) error - } - -Handlers can filter records, format them, or dispatch to multiple other Handlers. -This package implements a number of Handlers for common logging patterns that are -easily composed to create flexible, custom logging structures. - -Here's an example handler that prints logfmt output to Stdout: - - handler := log.StreamHandler(os.Stdout, log.LogfmtFormat()) - -Here's an example handler that defers to two other handlers. One handler only prints records -from the rpc package in logfmt to standard out. The other prints records at Error level -or above in JSON formatted output to the file /var/log/service.json - - handler := log.MultiHandler( - log.LvlFilterHandler(log.LvlError, log.Must.FileHandler("/var/log/service.json", log.JSONFormat())), - log.MatchFilterHandler("pkg", "app/rpc" log.StdoutHandler()) - ) - -# Logging File Names and Line Numbers - -This package implements three Handlers that add debugging information to the -context, CallerFileHandler, CallerFuncHandler and CallerStackHandler. Here's -an example that adds the source file and line number of each logging call to -the context. - - h := log.CallerFileHandler(log.StdoutHandler) - log.Root().SetHandler(h) - ... - log.Error("open file", "err", err) - -This will output a line that looks like: - - lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" caller=data.go:42 - -Here's an example that logs the call stack rather than just the call site. - - h := log.CallerStackHandler("%+v", log.StdoutHandler) - log.Root().SetHandler(h) - ... - log.Error("open file", "err", err) - -This will output a line that looks like: - - lvl=eror t=2014-05-02T16:07:23-0700 msg="open file" err="file not found" stack="[pkg/data.go:42 pkg/cmd/main.go]" - -The "%+v" format instructs the handler to include the path of the source file -relative to the compile time GOPATH. The github.com/go-stack/stack package -documents the full list of formatting verbs and modifiers available. - -# Custom Handlers - -The Handler interface is so simple that it's also trivial to write your own. Let's create an -example handler which tries to write to one handler, but if that fails it falls back to -writing to another handler and includes the error that it encountered when trying to write -to the primary. This might be useful when trying to log over a network socket, but if that -fails you want to log those records to a file on disk. - - type BackupHandler struct { - Primary Handler - Secondary Handler - } - - func (h *BackupHandler) Log (r *Record) error { - err := h.Primary.Log(r) - if err != nil { - r.Ctx = append(ctx, "primary_err", err) - return h.Secondary.Log(r) - } - return nil - } - -This pattern is so useful that a generic version that handles an arbitrary number of Handlers -is included as part of this library called FailoverHandler. - -# Logging Expensive Operations - -Sometimes, you want to log values that are extremely expensive to compute, but you don't want to pay -the price of computing them if you haven't turned up your logging level to a high level of detail. - -This package provides a simple type to annotate a logging operation that you want to be evaluated -lazily, just when it is about to be logged, so that it would not be evaluated if an upstream Handler -filters it out. Just wrap any function which takes no arguments with the log.Lazy type. For example: - - func factorRSAKey() (factors []int) { - // return the factors of a very large number - } - - log.Debug("factors", log.Lazy{factorRSAKey}) - -If this message is not logged for any reason (like logging at the Error level), then -factorRSAKey is never evaluated. - -# Dynamic context values - -The same log.Lazy mechanism can be used to attach context to a logger which you want to be -evaluated when the message is logged, but not when the logger is created. For example, let's imagine -a game where you have Player objects: - - type Player struct { - name string - alive bool - log.Logger - } - -You always want to log a player's name and whether they're alive or dead, so when you create the player -object, you might do: - - p := &Player{name: name, alive: true} - p.Logger = log.New("name", p.name, "alive", p.alive) - -Only now, even after a player has died, the logger will still report they are alive because the logging -context is evaluated when the logger was created. By using the Lazy wrapper, we can defer the evaluation -of whether the player is alive or not to each log message, so that the log records will reflect the player's -current state no matter when the log message is written: - - p := &Player{name: name, alive: true} - isAlive := func() bool { return p.alive } - player.Logger = log.New("name", p.name, "alive", log.Lazy{isAlive}) - -# Terminal Format - -If log15 detects that stdout is a terminal, it will configure the default -handler for it (which is log.StdoutHandler) to use TerminalFormat. This format -logs records nicely for your terminal, including color-coded output based -on log level. - -# Error Handling - -Becasuse log15 allows you to step around the type system, there are a few ways you can specify -invalid arguments to the logging functions. You could, for example, wrap something that is not -a zero-argument function with log.Lazy or pass a context key that is not a string. Since logging libraries -are typically the mechanism by which errors are reported, it would be onerous for the logging functions -to return errors. Instead, log15 handles errors by making these guarantees to you: - -- Any log record containing an error will still be printed with the error explained to you as part of the log record. - -- Any log record containing an error will include the context key LOG15_ERROR, enabling you to easily -(and if you like, automatically) detect if any of your logging calls are passing bad values. - -Understanding this, you might wonder why the Handler interface can return an error value in its Log method. Handlers -are encouraged to return errors only if they fail to write their log records out to an external source like if the -syslog daemon is not responding. This allows the construction of useful handlers which cope with those failures -like the FailoverHandler. - -# Library Use - -log15 is intended to be useful for library authors as a way to provide configurable logging to -users of their library. Best practice for use in a library is to always disable all output for your logger -by default and to provide a public Logger instance that consumers of your library can configure. Like so: - - package yourlib - - import "github.com/inconshreveable/log15" - - var Log = log.New() - - func init() { - Log.SetHandler(log.DiscardHandler()) - } - -Users of your library may then enable it if they like: - - import "github.com/inconshreveable/log15" - import "example.com/yourlib" - - func main() { - handler := // custom handler setup - yourlib.Log.SetHandler(handler) - } - -# Best practices attaching logger context - -The ability to attach context to a logger is a powerful one. Where should you do it and why? -I favor embedding a Logger directly into any persistent object in my application and adding -unique, tracing context keys to it. For instance, imagine I am writing a web browser: - - type Tab struct { - url string - render *RenderingContext - // ... - - Logger - } - - func NewTab(url string) *Tab { - return &Tab { - // ... - url: url, - - Logger: log.New("url", url), - } - } - -When a new tab is created, I assign a logger to it with the url of -the tab as context so it can easily be traced through the logs. -Now, whenever we perform any operation with the tab, we'll log with its -embedded logger and it will include the tab title automatically: - - tab.Debug("moved position", "idx", tab.idx) - -There's only one problem. What if the tab url changes? We could -use log.Lazy to make sure the current url is always written, but that -would mean that we couldn't trace a tab's full lifetime through our -logs after the user navigate to a new URL. - -Instead, think about what values to attach to your loggers the -same way you think about what to use as a key in a SQL database schema. -If it's possible to use a natural key that is unique for the lifetime of the -object, do so. But otherwise, log15's ext package has a handy RandId -function to let you generate what you might call "surrogate keys" -They're just random hex identifiers to use for tracing. Back to our -Tab example, we would prefer to set up our Logger like so: - - import logext "github.com/inconshreveable/log15/ext" - - t := &Tab { - // ... - url: url, - } - - t.Logger = log.New("id", logext.RandId(8), "url", log.Lazy{t.getUrl}) - return t - -Now we'll have a unique traceable identifier even across loading new urls, but -we'll still be able to see the tab's current url in the log messages. - -# Must - -For all Handler functions which can return an error, there is a version of that -function which will return no error but panics on failure. They are all available -on the Must object. For example: - - log.Must.FileHandler("/path", log.JSONFormat) - log.Must.NetHandler("tcp", ":1234", log.JSONFormat) - -# Inspiration and Credit - -All of the following excellent projects inspired the design of this library: - -code.google.com/p/log4go - -github.com/op/go-logging - -github.com/technoweenie/grohl - -github.com/Sirupsen/logrus - -github.com/kr/logfmt - -github.com/spacemonkeygo/spacelog - -golang's stdlib, notably io and net/http - -# The Name - -https://xkcd.com/927/ -*/ -package log diff --git a/log/format.go b/log/format.go index 2fd1f28558..5cbbe3341e 100644 --- a/log/format.go +++ b/log/format.go @@ -2,18 +2,15 @@ package log import ( "bytes" - "encoding/json" "fmt" "math/big" "reflect" "strconv" - "strings" - "sync" - "sync/atomic" "time" "unicode/utf8" "github.com/holiman/uint256" + "golang.org/x/exp/slog" ) const ( @@ -24,61 +21,19 @@ const ( termCtxMaxPadding = 40 ) -// ResetGlobalState resets the fieldPadding, which is useful for producing -// predictable output. -func ResetGlobalState() { - fieldPaddingLock.Lock() - fieldPadding = make(map[string]int) - fieldPaddingLock.Unlock() -} - -// locationTrims are trimmed for display to avoid unwieldy log lines. -var locationTrims = []string{ - "github.com/ethereum/go-ethereum/", -} - -// PrintOrigins sets or unsets log location (file:line) printing for terminal -// format output. -func PrintOrigins(print bool) { - locationEnabled.Store(print) - if print { - stackEnabled.Store(true) - } -} - -// stackEnabled is an atomic flag controlling whether the log handler needs -// to store the callsite stack. This is needed in case any handler wants to -// print locations (locationEnabled), use vmodule, or print full stacks (BacktraceAt). -var stackEnabled atomic.Bool - -// locationEnabled is an atomic flag controlling whether the terminal formatter -// should append the log locations too when printing entries. -var locationEnabled atomic.Bool - -// locationLength is the maxmimum path length encountered, which all logs are -// padded to to aid in alignment. -var locationLength atomic.Uint32 - -// fieldPadding is a global map with maximum field value lengths seen until now -// to allow padding log contexts in a bit smarter way. -var fieldPadding = make(map[string]int) - -// fieldPaddingLock is a global mutex protecting the field padding map. -var fieldPaddingLock sync.RWMutex - type Format interface { - Format(r *Record) []byte + Format(r slog.Record) []byte } // FormatFunc returns a new Format object which uses // the given function to perform record formatting. -func FormatFunc(f func(*Record) []byte) Format { +func FormatFunc(f func(slog.Record) []byte) Format { return formatFunc(f) } -type formatFunc func(*Record) []byte +type formatFunc func(slog.Record) []byte -func (f formatFunc) Format(r *Record) []byte { +func (f formatFunc) Format(r slog.Record) []byte { return f(r) } @@ -89,263 +44,100 @@ type TerminalStringer interface { TerminalString() string } -// TerminalFormat formats log records optimized for human readability on -// a terminal with color-coded level output and terser human friendly timestamp. -// This format should only be used for interactive programs or while developing. -// -// [LEVEL] [TIME] MESSAGE key=value key=value ... -// -// Example: -// -// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 -func TerminalFormat(usecolor bool) Format { - return FormatFunc(func(r *Record) []byte { - msg := escapeMessage(r.Msg) - var color = 0 - if usecolor { - switch r.Lvl { - case LvlCrit: - color = 35 - case LvlError: - color = 31 - case LvlWarn: - color = 33 - case LvlInfo: - color = 32 - case LvlDebug: - color = 36 - case LvlTrace: - color = 34 - } +func (h *TerminalHandler) TerminalFormat(r slog.Record, usecolor bool) []byte { + msg := escapeMessage(r.Message) + var color = 0 + if usecolor { + switch r.Level { + case LevelCrit: + color = 35 + case slog.LevelError: + color = 31 + case slog.LevelWarn: + color = 33 + case slog.LevelInfo: + color = 32 + case slog.LevelDebug: + color = 36 + case LevelTrace: + color = 34 } + } - b := &bytes.Buffer{} - lvl := r.Lvl.AlignedString() - if locationEnabled.Load() { - // Log origin printing was requested, format the location path and line number - location := fmt.Sprintf("%+v", r.Call) - for _, prefix := range locationTrims { - location = strings.TrimPrefix(location, prefix) - } - // Maintain the maximum location length for fancyer alignment - align := int(locationLength.Load()) - if align < len(location) { - align = len(location) - locationLength.Store(uint32(align)) - } - padding := strings.Repeat(" ", align-len(location)) + b := &bytes.Buffer{} + lvl := LevelAlignedString(r.Level) + if color > 0 { + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", color, lvl, r.Time.Format(termTimeFormat), msg) + } else { + fmt.Fprintf(b, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), msg) + } + // try to justify the log output for short messages + length := utf8.RuneCountInString(msg) + if r.NumAttrs() > 0 && length < termMsgJust { + b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length)) + } + // print the keys logfmt style + h.logfmt(b, r, color) - // Assemble and print the log heading - if color > 0 { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s|%s]%s %s ", color, lvl, r.Time.Format(termTimeFormat), location, padding, msg) - } else { - fmt.Fprintf(b, "%s[%s|%s]%s %s ", lvl, r.Time.Format(termTimeFormat), location, padding, msg) - } - } else { - if color > 0 { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", color, lvl, r.Time.Format(termTimeFormat), msg) - } else { - fmt.Fprintf(b, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), msg) - } - } - // try to justify the log output for short messages - length := utf8.RuneCountInString(msg) - if len(r.Ctx) > 0 && length < termMsgJust { - b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length)) - } - // print the keys logfmt style - logfmt(b, r.Ctx, color, true) - return b.Bytes() - }) + return b.Bytes() } -// LogfmtFormat prints records in logfmt format, an easy machine-parseable but human-readable -// format for key/value pairs. -// -// For more details see: http://godoc.org/github.com/kr/logfmt -func LogfmtFormat() Format { - return FormatFunc(func(r *Record) []byte { - common := []interface{}{r.KeyNames.Time, r.Time, r.KeyNames.Lvl, r.Lvl, r.KeyNames.Msg, r.Msg} - buf := &bytes.Buffer{} - logfmt(buf, append(common, r.Ctx...), 0, false) - return buf.Bytes() +func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color int) { + attrs := []slog.Attr{} + r.Attrs(func(attr slog.Attr) bool { + attrs = append(attrs, attr) + return true }) -} -func logfmt(buf *bytes.Buffer, ctx []interface{}, color int, term bool) { - for i := 0; i < len(ctx); i += 2 { + attrs = append(h.attrs, attrs...) + + for i, attr := range attrs { if i != 0 { buf.WriteByte(' ') } - k, ok := ctx[i].(string) - v := formatLogfmtValue(ctx[i+1], term) - if !ok { - k, v = errorKey, fmt.Sprintf("%+T is not a string key", ctx[i]) - } else { - k = escapeString(k) - } + key := escapeString(attr.Key) + rawVal := attr.Value.Any() + val := FormatLogfmtValue(rawVal, true) // XXX: we should probably check that all of your key bytes aren't invalid - fieldPaddingLock.RLock() - padding := fieldPadding[k] - fieldPaddingLock.RUnlock() + // TODO (jwasinger) above comment was from log15 code. what does it mean? check that key bytes are ascii characters? + padding := h.fieldPadding[key] - length := utf8.RuneCountInString(v) + length := utf8.RuneCountInString(val) if padding < length && length <= termCtxMaxPadding { padding = length - - fieldPaddingLock.Lock() - fieldPadding[k] = padding - fieldPaddingLock.Unlock() + h.fieldPadding[key] = padding } if color > 0 { - fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=", color, k) + fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=", color, key) } else { - buf.WriteString(k) + buf.WriteString(key) buf.WriteByte('=') } - buf.WriteString(v) - if i < len(ctx)-2 && padding > length { + buf.WriteString(val) + if i < r.NumAttrs()-1 && padding > length { buf.Write(bytes.Repeat([]byte{' '}, padding-length)) } } buf.WriteByte('\n') } -// JSONFormat formats log records as JSON objects separated by newlines. -// It is the equivalent of JSONFormatEx(false, true). -func JSONFormat() Format { - return JSONFormatEx(false, true) -} - -// JSONFormatOrderedEx formats log records as JSON arrays. If pretty is true, -// records will be pretty-printed. If lineSeparated is true, records -// will be logged with a new line between each record. -func JSONFormatOrderedEx(pretty, lineSeparated bool) Format { - jsonMarshal := json.Marshal - if pretty { - jsonMarshal = func(v interface{}) ([]byte, error) { - return json.MarshalIndent(v, "", " ") - } +// formatValue formats a value for serialization +func FormatLogfmtValue(value interface{}, term bool) (result string) { + if value == nil { + return "" } - return FormatFunc(func(r *Record) []byte { - props := map[string]interface{}{ - r.KeyNames.Time: r.Time, - r.KeyNames.Lvl: r.Lvl.String(), - r.KeyNames.Msg: r.Msg, - } - - ctx := make([]string, len(r.Ctx)) - for i := 0; i < len(r.Ctx); i += 2 { - if k, ok := r.Ctx[i].(string); ok { - ctx[i] = k - ctx[i+1] = formatLogfmtValue(r.Ctx[i+1], true) - } else { - props[errorKey] = fmt.Sprintf("%+T is not a string key,", r.Ctx[i]) - } - } - props[r.KeyNames.Ctx] = ctx - - b, err := jsonMarshal(props) - if err != nil { - b, _ = jsonMarshal(map[string]string{ - errorKey: err.Error(), - }) - return b - } - if lineSeparated { - b = append(b, '\n') - } - return b - }) -} - -// JSONFormatEx formats log records as JSON objects. If pretty is true, -// records will be pretty-printed. If lineSeparated is true, records -// will be logged with a new line between each record. -func JSONFormatEx(pretty, lineSeparated bool) Format { - jsonMarshal := json.Marshal - if pretty { - jsonMarshal = func(v interface{}) ([]byte, error) { - return json.MarshalIndent(v, "", " ") - } - } - - return FormatFunc(func(r *Record) []byte { - props := map[string]interface{}{ - r.KeyNames.Time: r.Time, - r.KeyNames.Lvl: r.Lvl.String(), - r.KeyNames.Msg: r.Msg, - } - - for i := 0; i < len(r.Ctx); i += 2 { - k, ok := r.Ctx[i].(string) - if !ok { - props[errorKey] = fmt.Sprintf("%+T is not a string key", r.Ctx[i]) - } else { - props[k] = formatJSONValue(r.Ctx[i+1]) - } - } - - b, err := jsonMarshal(props) - if err != nil { - b, _ = jsonMarshal(map[string]string{ - errorKey: err.Error(), - }) - return b - } - - if lineSeparated { - b = append(b, '\n') - } - - return b - }) -} - -func formatShared(value interface{}) (result interface{}) { defer func() { if err := recover(); err != nil { if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() { - result = "nil" + result = "" } else { panic(err) } } }() - switch v := value.(type) { - case time.Time: - return v.Format(timeFormat) - - case error: - return v.Error() - - case fmt.Stringer: - return v.String() - - default: - return v - } -} - -func formatJSONValue(value interface{}) interface{} { - value = formatShared(value) - switch value.(type) { - case int, int8, int16, int32, int64, float32, float64, uint, uint8, uint16, uint32, uint64, string: - return value - default: - return fmt.Sprintf("%+v", value) - } -} - -// formatValue formats a value for serialization -func formatLogfmtValue(value interface{}, term bool) string { - if value == nil { - return "nil" - } - switch v := value.(type) { case time.Time: // Performance optimization: No need for escaping since the provided @@ -375,8 +167,11 @@ func formatLogfmtValue(value interface{}, term bool) string { return escapeString(s.TerminalString()) } } - value = formatShared(value) switch v := value.(type) { + case error: + return escapeString(v.Error()) + case fmt.Stringer: + return escapeString(v.String()) case bool: return strconv.FormatBool(v) case float32: diff --git a/log/handler.go b/log/handler.go index 4a0cf578f6..ef1bcc1992 100644 --- a/log/handler.go +++ b/log/handler.go @@ -1,301 +1,25 @@ package log import ( + "context" "fmt" "io" - "net" - "os" + "math/big" "reflect" "sync" - "sync/atomic" + "time" - "github.com/go-stack/stack" + "github.com/holiman/uint256" + "golang.org/x/exp/slog" ) -// Handler defines where and how log records are written. -// A Logger prints its log records by writing to a Handler. -// Handlers are composable, providing you great flexibility in combining -// them to achieve the logging structure that suits your applications. -type Handler interface { - Log(r *Record) error -} - -// FuncHandler returns a Handler that logs records with the given -// function. -func FuncHandler(fn func(r *Record) error) Handler { - return funcHandler(fn) -} - -type funcHandler func(r *Record) error - -func (h funcHandler) Log(r *Record) error { - return h(r) -} - -// StreamHandler writes log records to an io.Writer -// with the given format. StreamHandler can be used -// to easily begin writing log records to other -// outputs. +// Lazy allows you to defer calculation of a logged value that is expensive +// to compute until it is certain that it must be evaluated with the given filters. // -// StreamHandler wraps itself with LazyHandler and SyncHandler -// to evaluate Lazy objects and perform safe concurrent writes. -func StreamHandler(wr io.Writer, fmtr Format) Handler { - h := FuncHandler(func(r *Record) error { - _, err := wr.Write(fmtr.Format(r)) - return err - }) - return LazyHandler(SyncHandler(h)) -} - -// SyncHandler can be wrapped around a handler to guarantee that -// only a single Log operation can proceed at a time. It's necessary -// for thread-safe concurrent writes. -func SyncHandler(h Handler) Handler { - var mu sync.Mutex - return FuncHandler(func(r *Record) error { - mu.Lock() - defer mu.Unlock() - - return h.Log(r) - }) -} - -// FileHandler returns a handler which writes log records to the give file -// using the given format. If the path -// already exists, FileHandler will append to the given file. If it does not, -// FileHandler will create the file with mode 0644. -func FileHandler(path string, fmtr Format) (Handler, error) { - f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) - if err != nil { - return nil, err - } - return closingHandler{f, StreamHandler(f, fmtr)}, nil -} - -// NetHandler opens a socket to the given address and writes records -// over the connection. -func NetHandler(network, addr string, fmtr Format) (Handler, error) { - conn, err := net.Dial(network, addr) - if err != nil { - return nil, err - } - - return closingHandler{conn, StreamHandler(conn, fmtr)}, nil -} - -// XXX: closingHandler is essentially unused at the moment -// it's meant for a future time when the Handler interface supports -// a possible Close() operation -type closingHandler struct { - io.WriteCloser - Handler -} - -func (h *closingHandler) Close() error { - return h.WriteCloser.Close() -} - -// CallerFileHandler returns a Handler that adds the line number and file of -// the calling function to the context with key "caller". -func CallerFileHandler(h Handler) Handler { - return FuncHandler(func(r *Record) error { - r.Ctx = append(r.Ctx, "caller", fmt.Sprint(r.Call)) - return h.Log(r) - }) -} - -// CallerFuncHandler returns a Handler that adds the calling function name to -// the context with key "fn". -func CallerFuncHandler(h Handler) Handler { - return FuncHandler(func(r *Record) error { - r.Ctx = append(r.Ctx, "fn", formatCall("%+n", r.Call)) - return h.Log(r) - }) -} - -// This function is here to please go vet on Go < 1.8. -func formatCall(format string, c stack.Call) string { - return fmt.Sprintf(format, c) -} - -// CallerStackHandler returns a Handler that adds a stack trace to the context -// with key "stack". The stack trace is formatted as a space separated list of -// call sites inside matching []'s. The most recent call site is listed first. -// Each call site is formatted according to format. See the documentation of -// package github.com/go-stack/stack for the list of supported formats. -func CallerStackHandler(format string, h Handler) Handler { - return FuncHandler(func(r *Record) error { - s := stack.Trace().TrimBelow(r.Call).TrimRuntime() - if len(s) > 0 { - r.Ctx = append(r.Ctx, "stack", fmt.Sprintf(format, s)) - } - return h.Log(r) - }) -} - -// FilterHandler returns a Handler that only writes records to the -// wrapped Handler if the given function evaluates true. For example, -// to only log records where the 'err' key is not nil: -// -// logger.SetHandler(FilterHandler(func(r *Record) bool { -// for i := 0; i < len(r.Ctx); i += 2 { -// if r.Ctx[i] == "err" { -// return r.Ctx[i+1] != nil -// } -// } -// return false -// }, h)) -func FilterHandler(fn func(r *Record) bool, h Handler) Handler { - return FuncHandler(func(r *Record) error { - if fn(r) { - return h.Log(r) - } - return nil - }) -} - -// MatchFilterHandler returns a Handler that only writes records -// to the wrapped Handler if the given key in the logged -// context matches the value. For example, to only log records -// from your ui package: -// -// log.MatchFilterHandler("pkg", "app/ui", log.StdoutHandler) -func MatchFilterHandler(key string, value interface{}, h Handler) Handler { - return FilterHandler(func(r *Record) (pass bool) { - switch key { - case r.KeyNames.Lvl: - return r.Lvl == value - case r.KeyNames.Time: - return r.Time == value - case r.KeyNames.Msg: - return r.Msg == value - } - - for i := 0; i < len(r.Ctx); i += 2 { - if r.Ctx[i] == key { - return r.Ctx[i+1] == value - } - } - return false - }, h) -} - -// LvlFilterHandler returns a Handler that only writes -// records which are less than the given verbosity -// level to the wrapped Handler. For example, to only -// log Error/Crit records: -// -// log.LvlFilterHandler(log.LvlError, log.StdoutHandler) -func LvlFilterHandler(maxLvl Lvl, h Handler) Handler { - return FilterHandler(func(r *Record) (pass bool) { - return r.Lvl <= maxLvl - }, h) -} - -// MultiHandler dispatches any write to each of its handlers. -// This is useful for writing different types of log information -// to different locations. For example, to log to a file and -// standard error: -// -// log.MultiHandler( -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StderrHandler) -func MultiHandler(hs ...Handler) Handler { - return FuncHandler(func(r *Record) error { - for _, h := range hs { - // what to do about failures? - h.Log(r) - } - return nil - }) -} - -// FailoverHandler writes all log records to the first handler -// specified, but will failover and write to the second handler if -// the first handler has failed, and so on for all handlers specified. -// For example you might want to log to a network socket, but failover -// to writing to a file if the network fails, and then to -// standard out if the file write fails: -// -// log.FailoverHandler( -// log.Must.NetHandler("tcp", ":9090", log.JSONFormat()), -// log.Must.FileHandler("/var/log/app.log", log.LogfmtFormat()), -// log.StdoutHandler) -// -// All writes that do not go to the first handler will add context with keys of -// the form "failover_err_{idx}" which explain the error encountered while -// trying to write to the handlers before them in the list. -func FailoverHandler(hs ...Handler) Handler { - return FuncHandler(func(r *Record) error { - var err error - for i, h := range hs { - err = h.Log(r) - if err == nil { - return nil - } - r.Ctx = append(r.Ctx, fmt.Sprintf("failover_err_%d", i), err) - } - - return err - }) -} - -// ChannelHandler writes all records to the given channel. -// It blocks if the channel is full. Useful for async processing -// of log messages, it's used by BufferedHandler. -func ChannelHandler(recs chan<- *Record) Handler { - return FuncHandler(func(r *Record) error { - recs <- r - return nil - }) -} - -// BufferedHandler writes all records to a buffered -// channel of the given size which flushes into the wrapped -// handler whenever it is available for writing. Since these -// writes happen asynchronously, all writes to a BufferedHandler -// never return an error and any errors from the wrapped handler are ignored. -func BufferedHandler(bufSize int, h Handler) Handler { - recs := make(chan *Record, bufSize) - go func() { - for m := range recs { - _ = h.Log(m) - } - }() - return ChannelHandler(recs) -} - -// LazyHandler writes all values to the wrapped handler after evaluating -// any lazy functions in the record's context. It is already wrapped -// around StreamHandler and SyslogHandler in this library, you'll only need -// it if you write your own Handler. -func LazyHandler(h Handler) Handler { - return FuncHandler(func(r *Record) error { - // go through the values (odd indices) and reassign - // the values of any lazy fn to the result of its execution - hadErr := false - for i := 1; i < len(r.Ctx); i += 2 { - lz, ok := r.Ctx[i].(Lazy) - if ok { - v, err := evaluateLazy(lz) - if err != nil { - hadErr = true - r.Ctx[i] = err - } else { - if cs, ok := v.(stack.CallStack); ok { - v = cs.TrimBelow(r.Call).TrimRuntime() - } - r.Ctx[i] = v - } - } - } - - if hadErr { - r.Ctx = append(r.Ctx, errorKey, "bad lazy") - } - - return h.Log(r) - }) +// You may wrap any function which takes no arguments to Lazy. It may return any +// number of values of any type. +type Lazy struct { + Fn interface{} } func evaluateLazy(lz Lazy) (interface{}, error) { @@ -325,51 +49,175 @@ func evaluateLazy(lz Lazy) (interface{}, error) { return values, nil } -// DiscardHandler reports success for all writes but does nothing. -// It is useful for dynamically disabling logging at runtime via -// a Logger's SetHandler method. -func DiscardHandler() Handler { - return FuncHandler(func(r *Record) error { - return nil +type discardHandler struct{} + +// DiscardHandler returns a no-op handler +func DiscardHandler() slog.Handler { + return &discardHandler{} +} + +func (h *discardHandler) Handle(_ context.Context, r slog.Record) error { + return nil +} + +func (h *discardHandler) Enabled(_ context.Context, level slog.Level) bool { + return false +} + +func (h *discardHandler) WithGroup(name string) slog.Handler { + panic("not implemented") +} + +func (h *discardHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &discardHandler{} +} + +type TerminalHandler struct { + mu sync.Mutex + wr io.Writer + lvl slog.Level + useColor bool + attrs []slog.Attr + // fieldPadding is a map with maximum field value lengths seen until now + // to allow padding log contexts in a bit smarter way. + fieldPadding map[string]int +} + +// NewTerminalHandler returns a handler which formats log records at all levels optimized for human readability on +// a terminal with color-coded level output and terser human friendly timestamp. +// This format should only be used for interactive programs or while developing. +// +// [LEVEL] [TIME] MESSAGE key=value key=value ... +// +// Example: +// +// [DBUG] [May 16 20:58:45] remove route ns=haproxy addr=127.0.0.1:50002 +func NewTerminalHandler(wr io.Writer, useColor bool) *TerminalHandler { + return NewTerminalHandlerWithLevel(wr, levelMaxVerbosity, useColor) +} + +// NewTerminalHandlerWithLevel returns the same handler as NewTerminalHandler but only outputs +// records which are less than or equal to the specified verbosity level. +func NewTerminalHandlerWithLevel(wr io.Writer, lvl slog.Level, useColor bool) *TerminalHandler { + return &TerminalHandler{ + wr: wr, + lvl: lvl, + useColor: useColor, + fieldPadding: make(map[string]int), + } +} + +func (h *TerminalHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.wr.Write(h.TerminalFormat(r, h.useColor)) + return nil +} + +func (h *TerminalHandler) Enabled(_ context.Context, level slog.Level) bool { + return level >= h.lvl +} + +func (h *TerminalHandler) WithGroup(name string) slog.Handler { + panic("not implemented") +} + +func (h *TerminalHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &TerminalHandler{ + wr: h.wr, + lvl: h.lvl, + useColor: h.useColor, + attrs: append(h.attrs, attrs...), + fieldPadding: make(map[string]int), + } +} + +// ResetFieldPadding zeroes the field-padding for all attribute pairs. +func (t *TerminalHandler) ResetFieldPadding() { + t.mu.Lock() + t.fieldPadding = make(map[string]int) + t.mu.Unlock() +} + +type leveler struct{ minLevel slog.Level } + +func (l *leveler) Level() slog.Level { + return l.minLevel +} + +func JSONHandler(wr io.Writer) slog.Handler { + return slog.NewJSONHandler(wr, &slog.HandlerOptions{ + ReplaceAttr: builtinReplaceJSON, }) } -// Must provides the following Handler creation functions -// which instead of returning an error parameter only return a Handler -// and panic on failure: FileHandler, NetHandler, SyslogHandler, SyslogNetHandler -var Must muster +// LogfmtHandler returns a handler which prints records in logfmt format, an easy machine-parseable but human-readable +// format for key/value pairs. +// +// For more details see: http://godoc.org/github.com/kr/logfmt +func LogfmtHandler(wr io.Writer) slog.Handler { + return slog.NewTextHandler(wr, &slog.HandlerOptions{ + ReplaceAttr: builtinReplaceLogfmt, + }) +} -func must(h Handler, err error) Handler { - if err != nil { - panic(err) +// LogfmtHandlerWithLevel returns the same handler as LogfmtHandler but it only outputs +// records which are less than or equal to the specified verbosity level. +func LogfmtHandlerWithLevel(wr io.Writer, level slog.Level) slog.Handler { + return slog.NewTextHandler(wr, &slog.HandlerOptions{ + ReplaceAttr: builtinReplaceLogfmt, + Level: &leveler{level}, + }) +} + +func builtinReplaceLogfmt(_ []string, attr slog.Attr) slog.Attr { + return builtinReplace(nil, attr, true) +} + +func builtinReplaceJSON(_ []string, attr slog.Attr) slog.Attr { + return builtinReplace(nil, attr, false) +} + +func builtinReplace(_ []string, attr slog.Attr, logfmt bool) slog.Attr { + switch attr.Key { + case slog.TimeKey: + if attr.Value.Kind() == slog.KindTime { + if logfmt { + return slog.String("t", attr.Value.Time().Format(timeFormat)) + } else { + return slog.Attr{Key: "t", Value: attr.Value} + } + } + case slog.LevelKey: + if l, ok := attr.Value.Any().(slog.Level); ok { + attr = slog.Any("lvl", LevelString(l)) + return attr + } } - return h -} -type muster struct{} - -func (m muster) FileHandler(path string, fmtr Format) Handler { - return must(FileHandler(path, fmtr)) -} - -func (m muster) NetHandler(network, addr string, fmtr Format) Handler { - return must(NetHandler(network, addr, fmtr)) -} - -// swapHandler wraps another handler that may be swapped out -// dynamically at runtime in a thread-safe fashion. -type swapHandler struct { - handler atomic.Value -} - -func (h *swapHandler) Log(r *Record) error { - return (*h.handler.Load().(*Handler)).Log(r) -} - -func (h *swapHandler) Swap(newHandler Handler) { - h.handler.Store(&newHandler) -} - -func (h *swapHandler) Get() Handler { - return *h.handler.Load().(*Handler) + switch v := attr.Value.Any().(type) { + case time.Time: + if logfmt { + attr = slog.String(attr.Key, v.Format(timeFormat)) + } + case *big.Int: + if v == nil { + attr.Value = slog.StringValue("") + } else { + attr.Value = slog.StringValue(v.String()) + } + case *uint256.Int: + if v == nil { + attr.Value = slog.StringValue("") + } else { + attr.Value = slog.StringValue(v.Dec()) + } + case fmt.Stringer: + if v == nil || (reflect.ValueOf(v).Kind() == reflect.Pointer && reflect.ValueOf(v).IsNil()) { + attr.Value = slog.StringValue("") + } else { + attr.Value = slog.StringValue(v.String()) + } + } + return attr } diff --git a/log/handler_glog.go b/log/handler_glog.go index afca0808b3..fb1e03c5b5 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -17,6 +17,7 @@ package log import ( + "context" "errors" "fmt" "regexp" @@ -25,54 +26,47 @@ import ( "strings" "sync" "sync/atomic" + + "golang.org/x/exp/slog" ) // errVmoduleSyntax is returned when a user vmodule pattern is invalid. var errVmoduleSyntax = errors.New("expect comma-separated list of filename=N") -// errTraceSyntax is returned when a user backtrace pattern is invalid. -var errTraceSyntax = errors.New("expect file.go:234") - // GlogHandler is a log handler that mimics the filtering features of Google's // glog logger: setting global log levels; overriding with callsite pattern // matches; and requesting backtraces at certain positions. type GlogHandler struct { - origin Handler // The origin handler this wraps + origin slog.Handler // The origin handler this wraps - level atomic.Uint32 // Current log level, atomically accessible - override atomic.Bool // Flag whether overrides are used, atomically accessible - backtrace atomic.Bool // Flag whether backtrace location is set + level atomic.Int32 // Current log level, atomically accessible + override atomic.Bool // Flag whether overrides are used, atomically accessible - patterns []pattern // Current list of patterns to override with - siteCache map[uintptr]Lvl // Cache of callsite pattern evaluations - location string // file:line location where to do a stackdump at - lock sync.RWMutex // Lock protecting the override pattern list + patterns []pattern // Current list of patterns to override with + siteCache map[uintptr]slog.Level // Cache of callsite pattern evaluations + location string // file:line location where to do a stackdump at + lock sync.RWMutex // Lock protecting the override pattern list } // NewGlogHandler creates a new log handler with filtering functionality similar // to Google's glog logger. The returned handler implements Handler. -func NewGlogHandler(h Handler) *GlogHandler { +func NewGlogHandler(h slog.Handler) *GlogHandler { return &GlogHandler{ origin: h, } } -// SetHandler updates the handler to write records to the specified sub-handler. -func (h *GlogHandler) SetHandler(nh Handler) { - h.origin = nh -} - // pattern contains a filter for the Vmodule option, holding a verbosity level // and a file pattern to match. type pattern struct { pattern *regexp.Regexp - level Lvl + level slog.Level } // Verbosity sets the glog verbosity ceiling. The verbosity of individual packages // and source files can be raised using Vmodule. -func (h *GlogHandler) Verbosity(level Lvl) { - h.level.Store(uint32(level)) +func (h *GlogHandler) Verbosity(level slog.Level) { + h.level.Store(int32(level)) } // Vmodule sets the glog verbosity pattern. @@ -108,11 +102,13 @@ func (h *GlogHandler) Vmodule(ruleset string) error { return errVmoduleSyntax } // Parse the level and if correct, assemble the filter rule - level, err := strconv.Atoi(parts[1]) + l, err := strconv.Atoi(parts[1]) if err != nil { return errVmoduleSyntax } - if level <= 0 { + level := FromLegacyLevel(l) + + if level == LevelCrit { continue // Ignore. It's harmless but no point in paying the overhead. } // Compile the rule pattern into a regular expression @@ -130,107 +126,84 @@ func (h *GlogHandler) Vmodule(ruleset string) error { matcher = matcher + "$" re, _ := regexp.Compile(matcher) - filter = append(filter, pattern{re, Lvl(level)}) + filter = append(filter, pattern{re, level}) } // Swap out the vmodule pattern for the new filter system h.lock.Lock() defer h.lock.Unlock() h.patterns = filter - h.siteCache = make(map[uintptr]Lvl) + h.siteCache = make(map[uintptr]slog.Level) h.override.Store(len(filter) != 0) - // Enable location storage (globally) - if len(h.patterns) > 0 { - stackEnabled.Store(true) - } + return nil } -// BacktraceAt sets the glog backtrace location. When set to a file and line -// number holding a logging statement, a stack trace will be written to the Info -// log whenever execution hits that statement. -// -// Unlike with Vmodule, the ".go" must be present. -func (h *GlogHandler) BacktraceAt(location string) error { - // Ensure the backtrace location contains two non-empty elements - parts := strings.Split(location, ":") - if len(parts) != 2 { - return errTraceSyntax - } - parts[0] = strings.TrimSpace(parts[0]) - parts[1] = strings.TrimSpace(parts[1]) - if len(parts[0]) == 0 || len(parts[1]) == 0 { - return errTraceSyntax - } - // Ensure the .go prefix is present and the line is valid - if !strings.HasSuffix(parts[0], ".go") { - return errTraceSyntax - } - if _, err := strconv.Atoi(parts[1]); err != nil { - return errTraceSyntax - } - // All seems valid - h.lock.Lock() - defer h.lock.Unlock() +func (h *GlogHandler) Enabled(ctx context.Context, lvl slog.Level) bool { + // fast-track skipping logging if override not enabled and the provided verbosity is above configured + return h.override.Load() || slog.Level(h.level.Load()) <= lvl +} - h.location = location - h.backtrace.Store(len(location) > 0) - // Enable location storage (globally) - stackEnabled.Store(true) - return nil +func (h *GlogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + h.lock.RLock() + siteCache := make(map[uintptr]slog.Level) + for k, v := range h.siteCache { + siteCache[k] = v + } + h.lock.RUnlock() + + patterns := []pattern{} + patterns = append(patterns, h.patterns...) + + res := GlogHandler{ + origin: h.origin.WithAttrs(attrs), + patterns: patterns, + siteCache: siteCache, + location: h.location, + } + + res.level.Store(h.level.Load()) + res.override.Store(h.override.Load()) + return &res +} + +func (h *GlogHandler) WithGroup(name string) slog.Handler { + panic("not implemented") } // Log implements Handler.Log, filtering a log record through the global, local // and backtrace filters, finally emitting it if either allow it through. -func (h *GlogHandler) Log(r *Record) error { - // If backtracing is requested, check whether this is the callsite - if h.backtrace.Load() { - // Everything below here is slow. Although we could cache the call sites the - // same way as for vmodule, backtracing is so rare it's not worth the extra - // complexity. - h.lock.RLock() - match := h.location == r.Call.String() - h.lock.RUnlock() - - if match { - // Callsite matched, raise the log level to info and gather the stacks - r.Lvl = LvlInfo - - buf := make([]byte, 1024*1024) - buf = buf[:runtime.Stack(buf, true)] - r.Msg += "\n\n" + string(buf) - } - } +func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error { // If the global log level allows, fast track logging - if h.level.Load() >= uint32(r.Lvl) { - return h.origin.Log(r) - } - // If no local overrides are present, fast track skipping - if !h.override.Load() { - return nil + if slog.Level(h.level.Load()) <= r.Level { + return h.origin.Handle(context.Background(), r) } + // Check callsite cache for previously calculated log levels h.lock.RLock() - lvl, ok := h.siteCache[r.Call.Frame().PC] + lvl, ok := h.siteCache[r.PC] h.lock.RUnlock() // If we didn't cache the callsite yet, calculate it if !ok { h.lock.Lock() + + fs := runtime.CallersFrames([]uintptr{r.PC}) + frame, _ := fs.Next() + for _, rule := range h.patterns { - if rule.pattern.MatchString(fmt.Sprintf("%+s", r.Call)) { - h.siteCache[r.Call.Frame().PC], lvl, ok = rule.level, rule.level, true - break + if rule.pattern.MatchString(fmt.Sprintf("%+s", frame.File)) { + h.siteCache[r.PC], lvl, ok = rule.level, rule.level, true } } // If no rule matched, remember to drop log the next time if !ok { - h.siteCache[r.Call.Frame().PC] = 0 + h.siteCache[r.PC] = 0 } h.lock.Unlock() } - if lvl >= r.Lvl { - return h.origin.Log(r) + if lvl <= r.Level { + return h.origin.Handle(context.Background(), r) } return nil } diff --git a/log/logger.go b/log/logger.go index 42e7e375d0..3e227745ad 100644 --- a/log/logger.go +++ b/log/logger.go @@ -1,294 +1,222 @@ package log import ( - "fmt" + "context" + "math" "os" + "runtime" "time" - "github.com/go-stack/stack" + "golang.org/x/exp/slog" ) -const timeKey = "t" -const lvlKey = "lvl" -const msgKey = "msg" -const ctxKey = "ctx" -const errorKey = "LOG15_ERROR" -const skipLevel = 2 - -type Lvl int +const errorKey = "LOG_ERROR" const ( - LvlCrit Lvl = iota - LvlError - LvlWarn - LvlInfo - LvlDebug - LvlTrace + legacyLevelCrit = iota + legacyLevelError + legacyLevelWarn + legacyLevelInfo + legacyLevelDebug + legacyLevelTrace ) -// AlignedString returns a 5-character string containing the name of a Lvl. -func (l Lvl) AlignedString() string { +const ( + levelMaxVerbosity slog.Level = math.MinInt + LevelTrace slog.Level = -8 + LevelDebug = slog.LevelDebug + LevelInfo = slog.LevelInfo + LevelWarn = slog.LevelWarn + LevelError = slog.LevelError + LevelCrit slog.Level = 12 + + // for backward-compatibility + LvlTrace = LevelTrace + LvlInfo = LevelInfo + LvlDebug = LevelDebug +) + +// convert from old Geth verbosity level constants +// to levels defined by slog +func FromLegacyLevel(lvl int) slog.Level { + switch lvl { + case legacyLevelCrit: + return LevelCrit + case legacyLevelError: + return slog.LevelError + case legacyLevelWarn: + return slog.LevelWarn + case legacyLevelInfo: + return slog.LevelInfo + case legacyLevelDebug: + return slog.LevelDebug + case legacyLevelTrace: + return LevelTrace + default: + break + } + + // TODO: should we allow use of custom levels or force them to match existing max/min if they fall outside the range as I am doing here? + if lvl > legacyLevelTrace { + return LevelTrace + } + return LevelCrit +} + +// LevelAlignedString returns a 5-character string containing the name of a Lvl. +func LevelAlignedString(l slog.Level) string { switch l { - case LvlTrace: + case LevelTrace: return "TRACE" - case LvlDebug: + case slog.LevelDebug: return "DEBUG" - case LvlInfo: + case slog.LevelInfo: return "INFO " - case LvlWarn: + case slog.LevelWarn: return "WARN " - case LvlError: + case slog.LevelError: return "ERROR" - case LvlCrit: + case LevelCrit: return "CRIT " default: - panic("bad level") + return "unknown level" } } -// String returns the name of a Lvl. -func (l Lvl) String() string { +// LevelString returns a 5-character string containing the name of a Lvl. +func LevelString(l slog.Level) string { switch l { - case LvlTrace: - return "trce" - case LvlDebug: - return "dbug" - case LvlInfo: + case LevelTrace: + return "trace" + case slog.LevelDebug: + return "debug" + case slog.LevelInfo: return "info" - case LvlWarn: + case slog.LevelWarn: return "warn" - case LvlError: + case slog.LevelError: return "eror" - case LvlCrit: + case LevelCrit: return "crit" default: - panic("bad level") + return "unknown" } } -// LvlFromString returns the appropriate Lvl from a string name. -// Useful for parsing command line args and configuration files. -func LvlFromString(lvlString string) (Lvl, error) { - switch lvlString { - case "trace", "trce": - return LvlTrace, nil - case "debug", "dbug": - return LvlDebug, nil - case "info": - return LvlInfo, nil - case "warn": - return LvlWarn, nil - case "error", "eror": - return LvlError, nil - case "crit": - return LvlCrit, nil - default: - return LvlDebug, fmt.Errorf("unknown level: %v", lvlString) - } -} - -// A Record is what a Logger asks its handler to write -type Record struct { - Time time.Time - Lvl Lvl - Msg string - Ctx []interface{} - Call stack.Call - KeyNames RecordKeyNames -} - -// RecordKeyNames gets stored in a Record when the write function is executed. -type RecordKeyNames struct { - Time string - Msg string - Lvl string - Ctx string -} - // A Logger writes key/value pairs to a Handler type Logger interface { - // New returns a new Logger that has this logger's context plus the given context + // With returns a new Logger that has this logger's attributes plus the given attributes + With(ctx ...interface{}) Logger + + // With returns a new Logger that has this logger's attributes plus the given attributes. Identical to 'With'. New(ctx ...interface{}) Logger - // GetHandler gets the handler associated with the logger. - GetHandler() Handler + // Log logs a message at the specified level with context key/value pairs + Log(level slog.Level, msg string, ctx ...interface{}) - // SetHandler updates the logger to write records to the specified handler. - SetHandler(h Handler) - - // Log a message at the trace level with context key/value pairs - // - // # Usage - // - // log.Trace("msg") - // log.Trace("msg", "key1", val1) - // log.Trace("msg", "key1", val1, "key2", val2) + // Trace log a message at the trace level with context key/value pairs Trace(msg string, ctx ...interface{}) - // Log a message at the debug level with context key/value pairs - // - // # Usage Examples - // - // log.Debug("msg") - // log.Debug("msg", "key1", val1) - // log.Debug("msg", "key1", val1, "key2", val2) + // Debug logs a message at the debug level with context key/value pairs Debug(msg string, ctx ...interface{}) - // Log a message at the info level with context key/value pairs - // - // # Usage Examples - // - // log.Info("msg") - // log.Info("msg", "key1", val1) - // log.Info("msg", "key1", val1, "key2", val2) + // Info logs a message at the info level with context key/value pairs Info(msg string, ctx ...interface{}) - // Log a message at the warn level with context key/value pairs - // - // # Usage Examples - // - // log.Warn("msg") - // log.Warn("msg", "key1", val1) - // log.Warn("msg", "key1", val1, "key2", val2) + // Warn logs a message at the warn level with context key/value pairs Warn(msg string, ctx ...interface{}) - // Log a message at the error level with context key/value pairs - // - // # Usage Examples - // - // log.Error("msg") - // log.Error("msg", "key1", val1) - // log.Error("msg", "key1", val1, "key2", val2) + // Error logs a message at the error level with context key/value pairs Error(msg string, ctx ...interface{}) - // Log a message at the crit level with context key/value pairs, and then exit. - // - // # Usage Examples - // - // log.Crit("msg") - // log.Crit("msg", "key1", val1) - // log.Crit("msg", "key1", val1, "key2", val2) + // Crit logs a message at the crit level with context key/value pairs, and exits Crit(msg string, ctx ...interface{}) + + // Write logs a message at the specified level + Write(level slog.Level, msg string, attrs ...any) } type logger struct { - ctx []interface{} - h *swapHandler + inner *slog.Logger } -func (l *logger) write(msg string, lvl Lvl, ctx []interface{}, skip int) { - record := &Record{ - Time: time.Now(), - Lvl: lvl, - Msg: msg, - Ctx: newContext(l.ctx, ctx), - KeyNames: RecordKeyNames{ - Time: timeKey, - Msg: msgKey, - Lvl: lvlKey, - Ctx: ctxKey, - }, +// NewLogger returns a logger with the specified handler set +func NewLogger(h slog.Handler) Logger { + return &logger{ + slog.New(h), } - if stackEnabled.Load() { - record.Call = stack.Caller(skip) +} + +// write logs a message at the specified level: +func (l *logger) Write(level slog.Level, msg string, attrs ...any) { + if !l.inner.Enabled(context.Background(), level) { + return } - l.h.Log(record) -} -func (l *logger) New(ctx ...interface{}) Logger { - child := &logger{newContext(l.ctx, ctx), new(swapHandler)} - child.SetHandler(l.h) - return child -} + var pcs [1]uintptr + runtime.Callers(3, pcs[:]) -func newContext(prefix []interface{}, suffix []interface{}) []interface{} { - normalizedSuffix := normalize(suffix) - newCtx := make([]interface{}, len(prefix)+len(normalizedSuffix)) - n := copy(newCtx, prefix) - copy(newCtx[n:], normalizedSuffix) - return newCtx -} + if len(attrs)%2 != 0 { + attrs = append(attrs, nil, errorKey, "Normalized odd number of arguments by adding nil") + } -func (l *logger) Trace(msg string, ctx ...interface{}) { - l.write(msg, LvlTrace, ctx, skipLevel) -} - -func (l *logger) Debug(msg string, ctx ...interface{}) { - l.write(msg, LvlDebug, ctx, skipLevel) -} - -func (l *logger) Info(msg string, ctx ...interface{}) { - l.write(msg, LvlInfo, ctx, skipLevel) -} - -func (l *logger) Warn(msg string, ctx ...interface{}) { - l.write(msg, LvlWarn, ctx, skipLevel) -} - -func (l *logger) Error(msg string, ctx ...interface{}) { - l.write(msg, LvlError, ctx, skipLevel) -} - -func (l *logger) Crit(msg string, ctx ...interface{}) { - l.write(msg, LvlCrit, ctx, skipLevel) - os.Exit(1) -} - -func (l *logger) GetHandler() Handler { - return l.h.Get() -} - -func (l *logger) SetHandler(h Handler) { - l.h.Swap(h) -} - -func normalize(ctx []interface{}) []interface{} { - // if the caller passed a Ctx object, then expand it - if len(ctx) == 1 { - if ctxMap, ok := ctx[0].(Ctx); ok { - ctx = ctxMap.toArray() + // evaluate lazy values + var hadErr bool + for i := 1; i < len(attrs); i += 2 { + lz, ok := attrs[i].(Lazy) + if ok { + v, err := evaluateLazy(lz) + if err != nil { + hadErr = true + attrs[i] = err + } else { + attrs[i] = v + } } } - // ctx needs to be even because it's a series of key/value pairs - // no one wants to check for errors on logging functions, - // so instead of erroring on bad input, we'll just make sure - // that things are the right length and users can fix bugs - // when they see the output looks wrong - if len(ctx)%2 != 0 { - ctx = append(ctx, nil, errorKey, "Normalized odd number of arguments by adding nil") + if hadErr { + attrs = append(attrs, errorKey, "bad lazy") } - return ctx + r := slog.NewRecord(time.Now(), level, msg, pcs[0]) + r.Add(attrs...) + l.inner.Handler().Handle(context.Background(), r) } -// Lazy allows you to defer calculation of a logged value that is expensive -// to compute until it is certain that it must be evaluated with the given filters. -// -// Lazy may also be used in conjunction with a Logger's New() function -// to generate a child logger which always reports the current value of changing -// state. -// -// You may wrap any function which takes no arguments to Lazy. It may return any -// number of values of any type. -type Lazy struct { - Fn interface{} +func (l *logger) Log(level slog.Level, msg string, attrs ...any) { + l.Write(level, msg, attrs...) } -// Ctx is a map of key/value pairs to pass as context to a log function -// Use this only if you really need greater safety around the arguments you pass -// to the logging functions. -type Ctx map[string]interface{} - -func (c Ctx) toArray() []interface{} { - arr := make([]interface{}, len(c)*2) - - i := 0 - for k, v := range c { - arr[i] = k - arr[i+1] = v - i += 2 - } - - return arr +func (l *logger) With(ctx ...interface{}) Logger { + return &logger{l.inner.With(ctx...)} +} + +func (l *logger) New(ctx ...interface{}) Logger { + return l.With(ctx...) +} + +func (l *logger) Trace(msg string, ctx ...interface{}) { + l.Write(LevelTrace, msg, ctx...) +} + +func (l *logger) Debug(msg string, ctx ...interface{}) { + l.Write(slog.LevelDebug, msg, ctx...) +} + +func (l *logger) Info(msg string, ctx ...interface{}) { + l.Write(slog.LevelInfo, msg, ctx...) +} + +func (l *logger) Warn(msg string, ctx ...any) { + l.Write(slog.LevelWarn, msg, ctx...) +} + +func (l *logger) Error(msg string, ctx ...interface{}) { + l.Write(slog.LevelError, msg, ctx...) +} + +func (l *logger) Crit(msg string, ctx ...interface{}) { + l.Write(LevelCrit, msg, ctx...) + os.Exit(1) } diff --git a/log/logger_test.go b/log/logger_test.go index 2e59b3fdf0..fca1f1680f 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -5,48 +5,18 @@ import ( "os" "strings" "testing" -) -// TestLoggingWithTrace checks that if BackTraceAt is set, then the -// gloghandler is capable of spitting out a stacktrace -func TestLoggingWithTrace(t *testing.T) { - defer stackEnabled.Store(stackEnabled.Load()) - out := new(bytes.Buffer) - logger := New() - { - glog := NewGlogHandler(StreamHandler(out, TerminalFormat(false))) - glog.Verbosity(LvlTrace) - if err := glog.BacktraceAt("logger_test.go:24"); err != nil { - t.Fatal(err) - } - logger.SetHandler(glog) - } - logger.Trace("a message", "foo", "bar") // Will be bumped to INFO - have := out.String() - if !strings.HasPrefix(have, "INFO") { - t.Fatalf("backtraceat should bump level to info: %s", have) - } - // The timestamp is locale-dependent, so we want to trim that off - // "INFO [01-01|00:00:00.000] a messag ..." -> "a messag..." - have = strings.Split(have, "]")[1] - wantPrefix := " a message\n\ngoroutine" - if !strings.HasPrefix(have, wantPrefix) { - t.Errorf("\nhave: %q\nwant: %q\n", have, wantPrefix) - } -} + "golang.org/x/exp/slog" +) // TestLoggingWithVmodule checks that vmodule works. func TestLoggingWithVmodule(t *testing.T) { - defer stackEnabled.Store(stackEnabled.Load()) out := new(bytes.Buffer) - logger := New() - { - glog := NewGlogHandler(StreamHandler(out, TerminalFormat(false))) - glog.Verbosity(LvlCrit) - logger.SetHandler(glog) - logger.Warn("This should not be seen", "ignored", "true") - glog.Vmodule("logger_test.go=5") - } + glog := NewGlogHandler(NewTerminalHandlerWithLevel(out, LevelTrace, false)) + glog.Verbosity(LevelCrit) + logger := NewLogger(glog) + logger.Warn("This should not be seen", "ignored", "true") + glog.Vmodule("logger_test.go=5") logger.Trace("a message", "foo", "bar") have := out.String() // The timestamp is locale-dependent, so we want to trim that off @@ -58,8 +28,24 @@ func TestLoggingWithVmodule(t *testing.T) { } } +func TestTerminalHandlerWithAttrs(t *testing.T) { + out := new(bytes.Buffer) + glog := NewGlogHandler(NewTerminalHandlerWithLevel(out, LevelTrace, false).WithAttrs([]slog.Attr{slog.String("baz", "bat")})) + glog.Verbosity(LevelTrace) + logger := NewLogger(glog) + logger.Trace("a message", "foo", "bar") + have := out.String() + // The timestamp is locale-dependent, so we want to trim that off + // "INFO [01-01|00:00:00.000] a messag ..." -> "a messag..." + have = strings.Split(have, "]")[1] + want := " a message baz=bat foo=bar\n" + if have != want { + t.Errorf("\nhave: %q\nwant: %q\n", have, want) + } +} + func BenchmarkTraceLogging(b *testing.B) { - Root().SetHandler(LvlFilterHandler(LvlInfo, StreamHandler(os.Stderr, TerminalFormat(true)))) + SetDefault(NewLogger(NewTerminalHandler(os.Stderr, true))) b.ResetTimer() for i := 0; i < b.N; i++ { Trace("a message", "v", i) diff --git a/log/root.go b/log/root.go index 5a41723c3e..71040fff47 100644 --- a/log/root.go +++ b/log/root.go @@ -2,31 +2,33 @@ package log import ( "os" + "sync/atomic" + + "golang.org/x/exp/slog" ) -var ( - root = &logger{[]interface{}{}, new(swapHandler)} - StdoutHandler = StreamHandler(os.Stdout, LogfmtFormat()) - StderrHandler = StreamHandler(os.Stderr, LogfmtFormat()) -) +var root atomic.Value func init() { - root.SetHandler(DiscardHandler()) + defaultLogger := &logger{slog.New(DiscardHandler())} + SetDefault(defaultLogger) } -// New returns a new logger with the given context. -// New is a convenient alias for Root().New -func New(ctx ...interface{}) Logger { - return root.New(ctx...) +// SetDefault sets the default global logger +func SetDefault(l Logger) { + root.Store(l) + if lg, ok := l.(*logger); ok { + slog.SetDefault(lg.inner) + } } // Root returns the root logger func Root() Logger { - return root + return root.Load().(Logger) } // The following functions bypass the exported logger methods (logger.Debug, -// etc.) to keep the call depth the same for all paths to logger.write so +// etc.) to keep the call depth the same for all paths to logger.Write so // runtime.Caller(2) always refers to the call site in client code. // Trace is a convenient alias for Root().Trace @@ -39,7 +41,7 @@ func Root() Logger { // log.Trace("msg", "key1", val1) // log.Trace("msg", "key1", val1, "key2", val2) func Trace(msg string, ctx ...interface{}) { - root.write(msg, LvlTrace, ctx, skipLevel) + Root().Write(LevelTrace, msg, ctx...) } // Debug is a convenient alias for Root().Debug @@ -52,7 +54,7 @@ func Trace(msg string, ctx ...interface{}) { // log.Debug("msg", "key1", val1) // log.Debug("msg", "key1", val1, "key2", val2) func Debug(msg string, ctx ...interface{}) { - root.write(msg, LvlDebug, ctx, skipLevel) + Root().Write(slog.LevelDebug, msg, ctx...) } // Info is a convenient alias for Root().Info @@ -65,7 +67,7 @@ func Debug(msg string, ctx ...interface{}) { // log.Info("msg", "key1", val1) // log.Info("msg", "key1", val1, "key2", val2) func Info(msg string, ctx ...interface{}) { - root.write(msg, LvlInfo, ctx, skipLevel) + Root().Write(slog.LevelInfo, msg, ctx...) } // Warn is a convenient alias for Root().Warn @@ -78,7 +80,7 @@ func Info(msg string, ctx ...interface{}) { // log.Warn("msg", "key1", val1) // log.Warn("msg", "key1", val1, "key2", val2) func Warn(msg string, ctx ...interface{}) { - root.write(msg, LvlWarn, ctx, skipLevel) + Root().Write(slog.LevelWarn, msg, ctx...) } // Error is a convenient alias for Root().Error @@ -91,7 +93,7 @@ func Warn(msg string, ctx ...interface{}) { // log.Error("msg", "key1", val1) // log.Error("msg", "key1", val1, "key2", val2) func Error(msg string, ctx ...interface{}) { - root.write(msg, LvlError, ctx, skipLevel) + Root().Write(slog.LevelError, msg, ctx...) } // Crit is a convenient alias for Root().Crit @@ -104,15 +106,12 @@ func Error(msg string, ctx ...interface{}) { // log.Crit("msg", "key1", val1) // log.Crit("msg", "key1", val1, "key2", val2) func Crit(msg string, ctx ...interface{}) { - root.write(msg, LvlCrit, ctx, skipLevel) + Root().Write(LevelCrit, msg, ctx...) os.Exit(1) } -// Output is a convenient alias for write, allowing for the modification of -// the calldepth (number of stack frames to skip). -// calldepth influences the reported line number of the log message. -// A calldepth of zero reports the immediate caller of Output. -// Non-zero calldepth skips as many stack frames. -func Output(msg string, lvl Lvl, calldepth int, ctx ...interface{}) { - root.write(msg, lvl, ctx, calldepth+skipLevel) +// New returns a new logger with the given context. +// New is a convenient alias for Root().New +func New(ctx ...interface{}) Logger { + return Root().With(ctx...) } diff --git a/log/syslog.go b/log/syslog.go deleted file mode 100644 index 451d831b6d..0000000000 --- a/log/syslog.go +++ /dev/null @@ -1,58 +0,0 @@ -//go:build !windows && !plan9 -// +build !windows,!plan9 - -package log - -import ( - "log/syslog" - "strings" -) - -// SyslogHandler opens a connection to the system syslog daemon by calling -// syslog.New and writes all records to it. -func SyslogHandler(priority syslog.Priority, tag string, fmtr Format) (Handler, error) { - wr, err := syslog.New(priority, tag) - return sharedSyslog(fmtr, wr, err) -} - -// SyslogNetHandler opens a connection to a log daemon over the network and writes -// all log records to it. -func SyslogNetHandler(net, addr string, priority syslog.Priority, tag string, fmtr Format) (Handler, error) { - wr, err := syslog.Dial(net, addr, priority, tag) - return sharedSyslog(fmtr, wr, err) -} - -func sharedSyslog(fmtr Format, sysWr *syslog.Writer, err error) (Handler, error) { - if err != nil { - return nil, err - } - h := FuncHandler(func(r *Record) error { - var syslogFn = sysWr.Info - switch r.Lvl { - case LvlCrit: - syslogFn = sysWr.Crit - case LvlError: - syslogFn = sysWr.Err - case LvlWarn: - syslogFn = sysWr.Warning - case LvlInfo: - syslogFn = sysWr.Info - case LvlDebug: - syslogFn = sysWr.Debug - case LvlTrace: - syslogFn = func(m string) error { return nil } // There's no syslog level for trace - } - - s := strings.TrimSpace(string(fmtr.Format(r))) - return syslogFn(s) - }) - return LazyHandler(&closingHandler{sysWr, h}), nil -} - -func (m muster) SyslogHandler(priority syslog.Priority, tag string, fmtr Format) Handler { - return must(SyslogHandler(priority, tag, fmtr)) -} - -func (m muster) SyslogNetHandler(net, addr string, priority syslog.Priority, tag string, fmtr Format) Handler { - return must(SyslogNetHandler(net, addr, priority, tag, fmtr)) -} diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go index 7b29e63dfc..13336cd83c 100644 --- a/miner/stress/clique/main.go +++ b/miner/stress/clique/main.go @@ -45,7 +45,7 @@ import ( ) func main() { - log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) fdlimit.Raise(2048) // Generate a batch of accounts to seal and fund with diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 5add9cefa1..53ecb1bc6e 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -557,12 +557,7 @@ func startLocalhostV4(t *testing.T, cfg Config) *UDPv4 { // Prefix logs with node ID. lprefix := fmt.Sprintf("(%s)", ln.ID().TerminalString()) - lfmt := log.TerminalFormat(false) - cfg.Log = testlog.Logger(t, log.LvlTrace) - cfg.Log.SetHandler(log.FuncHandler(func(r *log.Record) error { - t.Logf("%s %s", lprefix, lfmt.Format(r)) - return nil - })) + cfg.Log = testlog.Logger(t, log.LevelTrace).With("node-id", lprefix) // Listen. socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{127, 0, 0, 1}}) diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 880b71a991..18d8aeac6d 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -79,12 +79,7 @@ func startLocalhostV5(t *testing.T, cfg Config) *UDPv5 { // Prefix logs with node ID. lprefix := fmt.Sprintf("(%s)", ln.ID().TerminalString()) - lfmt := log.TerminalFormat(false) - cfg.Log = testlog.Logger(t, log.LvlTrace) - cfg.Log.SetHandler(log.FuncHandler(func(r *log.Record) error { - t.Logf("%s %s", lprefix, lfmt.Format(r)) - return nil - })) + cfg.Log = testlog.Logger(t, log.LevelTrace).With("node-id", lprefix) // Listen. socket, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IP{127, 0, 0, 1}}) diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 5ac3379393..63cc4936c1 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -41,6 +41,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rpc" "github.com/gorilla/websocket" + "golang.org/x/exp/slog" ) func init() { @@ -375,9 +376,11 @@ type execNodeConfig struct { func initLogging() { // Initialize the logging by default first. - glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.LogfmtFormat())) - glogger.Verbosity(log.LvlInfo) - log.Root().SetHandler(glogger) + var innerHandler slog.Handler + innerHandler = slog.NewTextHandler(os.Stderr, nil) + glogger := log.NewGlogHandler(innerHandler) + glogger.Verbosity(log.LevelInfo) + log.SetDefault(log.NewLogger(glogger)) confEnv := os.Getenv(envNodeConfig) if confEnv == "" { @@ -395,14 +398,15 @@ func initLogging() { } writer = logWriter } - var verbosity = log.LvlInfo - if conf.Node.LogVerbosity <= log.LvlTrace && conf.Node.LogVerbosity >= log.LvlCrit { - verbosity = conf.Node.LogVerbosity + var verbosity = log.LevelInfo + if conf.Node.LogVerbosity <= log.LevelTrace && conf.Node.LogVerbosity >= log.LevelCrit { + verbosity = log.FromLegacyLevel(int(conf.Node.LogVerbosity)) } // Reinitialize the logger - glogger = log.NewGlogHandler(log.StreamHandler(writer, log.TerminalFormat(true))) + innerHandler = log.NewTerminalHandler(writer, true) + glogger = log.NewGlogHandler(innerHandler) glogger.Verbosity(verbosity) - log.Root().SetHandler(glogger) + log.SetDefault(log.NewLogger(glogger)) } // execP2PNode starts a simulation node when the current binary is executed with diff --git a/p2p/simulations/adapters/types.go b/p2p/simulations/adapters/types.go index 098759599c..fb8463d221 100644 --- a/p2p/simulations/adapters/types.go +++ b/p2p/simulations/adapters/types.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rpc" "github.com/gorilla/websocket" + "golang.org/x/exp/slog" ) // Node represents a node in a simulation network which is created by a @@ -129,7 +130,7 @@ type NodeConfig struct { // LogVerbosity is the log verbosity of the p2p node at runtime. // // The default verbosity is INFO. - LogVerbosity log.Lvl + LogVerbosity slog.Level } // nodeConfigJSON is used to encode and decode NodeConfig as JSON by encoding @@ -197,7 +198,7 @@ func (n *NodeConfig) UnmarshalJSON(data []byte) error { n.Port = confJSON.Port n.EnableMsgEvents = confJSON.EnableMsgEvents n.LogFile = confJSON.LogFile - n.LogVerbosity = log.Lvl(confJSON.LogVerbosity) + n.LogVerbosity = slog.Level(confJSON.LogVerbosity) return nil } diff --git a/p2p/simulations/examples/ping-pong.go b/p2p/simulations/examples/ping-pong.go index f6cf5113a6..70b35ad777 100644 --- a/p2p/simulations/examples/ping-pong.go +++ b/p2p/simulations/examples/ping-pong.go @@ -41,7 +41,7 @@ func main() { flag.Parse() // set the log level to Trace - log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, false))) // register a single ping-pong service services := map[string]adapters.LifecycleConstructor{ diff --git a/p2p/simulations/http_test.go b/p2p/simulations/http_test.go index 05e43238ab..c53a49797b 100644 --- a/p2p/simulations/http_test.go +++ b/p2p/simulations/http_test.go @@ -37,14 +37,14 @@ import ( "github.com/ethereum/go-ethereum/p2p/simulations/adapters" "github.com/ethereum/go-ethereum/rpc" "github.com/mattn/go-colorable" + "golang.org/x/exp/slog" ) func TestMain(m *testing.M) { loglevel := flag.Int("loglevel", 2, "verbosity of logs") flag.Parse() - log.PrintOrigins(true) - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(colorable.NewColorableStderr(), slog.Level(*loglevel), true))) os.Exit(m.Run()) } diff --git a/signer/core/auditlog.go b/signer/core/auditlog.go index a0b292bf71..d2207c9eb8 100644 --- a/signer/core/auditlog.go +++ b/signer/core/auditlog.go @@ -19,12 +19,14 @@ package core import ( "context" "encoding/json" + "os" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/signer/core/apitypes" + "golang.org/x/exp/slog" ) type AuditLogger struct { @@ -113,12 +115,13 @@ func (l *AuditLogger) Version(ctx context.Context) (string, error) { } func NewAuditLogger(path string, api ExternalAPI) (*AuditLogger, error) { - l := log.New("api", "signer") - handler, err := log.FileHandler(path, log.LogfmtFormat()) + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) if err != nil { return nil, err } - l.SetHandler(handler) + + handler := slog.NewTextHandler(f, nil) + l := log.NewLogger(handler).With("api", "signer") l.Info("Configured", "audit log", path) return &AuditLogger{l, api}, nil } diff --git a/signer/storage/aes_gcm_storage_test.go b/signer/storage/aes_gcm_storage_test.go index 74d407e431..a223b1a6b4 100644 --- a/signer/storage/aes_gcm_storage_test.go +++ b/signer/storage/aes_gcm_storage_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/mattn/go-colorable" + "golang.org/x/exp/slog" ) func TestEncryption(t *testing.T) { @@ -92,7 +93,7 @@ func TestFileStorage(t *testing.T) { } func TestEnd2End(t *testing.T) { t.Parallel() - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(colorable.NewColorableStderr(), slog.LevelInfo, true))) d := t.TempDir() @@ -115,7 +116,7 @@ func TestSwappedKeys(t *testing.T) { t.Parallel() // It should not be possible to swap the keys/values, so that // K1:V1, K2:V2 can be swapped into K1:V2, K2:V1 - log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true)))) + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(colorable.NewColorableStderr(), slog.LevelInfo, true))) d := t.TempDir() From ab0eb46a8466f12f8a53f4591a9563d86d9af9f5 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 29 Nov 2023 16:07:51 +0800 Subject: [PATCH 15/43] core/state: make stateobject.create selfcontain (#28459) --- core/state/state_object.go | 6 +++++- core/state/statedb.go | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/state/state_object.go b/core/state/state_object.go index fc66b48114..9383b98e44 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -98,7 +98,10 @@ func (s *stateObject) empty() bool { // newObject creates a state object. func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *stateObject { - origin := acct + var ( + origin = acct + created = acct == nil // true if the account was not existent + ) if acct == nil { acct = types.NewEmptyStateAccount() } @@ -111,6 +114,7 @@ func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *s originStorage: make(Storage), pendingStorage: make(Storage), dirtyStorage: make(Storage), + created: created, } } diff --git a/core/state/statedb.go b/core/state/statedb.go index 674227857c..905944cbb5 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -658,9 +658,6 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) delete(s.accountsOrigin, prev.address) delete(s.storagesOrigin, prev.address) } - - newobj.created = true - s.setStateObject(newobj) if prev != nil && !prev.deleted { return newobj, prev From fa0df76f3cfd186a1f06f2b80aa5dbb89555b009 Mon Sep 17 00:00:00 2001 From: Maciej Kulawik <10907694+magicxyyz@users.noreply.github.com> Date: Thu, 30 Nov 2023 09:50:48 +0000 Subject: [PATCH 16/43] trie/triedb/hashdb: take lock around access to dirties cache (#28542) Add read locking of db lock around access to dirties cache in hashdb.Database to prevent data race versus hashdb.Database.dereference which can modify the dirities map by deleting an item. Fixes #28541 --------- Co-authored-by: Gary Rong --- trie/database.go | 11 ------ trie/triedb/hashdb/database.go | 69 ++++++++++++---------------------- 2 files changed, 23 insertions(+), 57 deletions(-) diff --git a/trie/database.go b/trie/database.go index 321b4f8955..e20f7ef903 100644 --- a/trie/database.go +++ b/trie/database.go @@ -240,17 +240,6 @@ func (db *Database) Dereference(root common.Hash) error { return nil } -// Node retrieves the rlp-encoded node blob with provided node hash. It's -// only supported by hash-based database and will return an error for others. -// Note, this function should be deprecated once ETH66 is deprecated. -func (db *Database) Node(hash common.Hash) ([]byte, error) { - hdb, ok := db.backend.(*hashdb.Database) - if !ok { - return nil, errors.New("not supported") - } - return hdb.Node(hash) -} - // Recover rollbacks the database to a specified historical point. The state is // supported as the rollback destination only if it's canonical state and the // corresponding trie histories are existent. It's only supported by path-based diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index 764ab24ec8..e45ccdba32 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -82,11 +82,6 @@ var Defaults = &Config{ // Database is an intermediate write layer between the trie data structures and // the disk database. The aim is to accumulate trie writes in-memory and only // periodically flush a couple tries to disk, garbage collecting the remainder. -// -// Note, the trie Database is **not** thread safe in its mutations, but it **is** -// thread safe in providing individual, independent node access. The rationale -// behind this split design is to provide read access to RPC handlers and sync -// servers even while the trie is executing expensive garbage collection. type Database struct { diskdb ethdb.Database // Persistent storage for matured trie nodes resolver ChildResolver // The handler to resolve children of nodes @@ -113,7 +108,7 @@ type Database struct { // cachedNode is all the information we know about a single cached trie node // in the memory database write layer. type cachedNode struct { - node []byte // Encoded node blob + node []byte // Encoded node blob, immutable parents uint32 // Number of live nodes referencing this one external map[common.Hash]struct{} // The set of external children flushPrev common.Hash // Previous node in the flush-list @@ -152,9 +147,9 @@ func New(diskdb ethdb.Database, config *Config, resolver ChildResolver) *Databas } } -// insert inserts a simplified trie node into the memory database. -// All nodes inserted by this function will be reference tracked -// and in theory should only used for **trie nodes** insertion. +// insert inserts a trie node into the memory database. All nodes inserted by +// this function will be reference tracked. This function assumes the lock is +// already held. func (db *Database) insert(hash common.Hash, node []byte) { // If the node's already cached, skip if _, ok := db.dirties[hash]; ok { @@ -183,9 +178,9 @@ func (db *Database) insert(hash common.Hash, node []byte) { db.dirtiesSize += common.StorageSize(common.HashLength + len(node)) } -// Node retrieves an encoded cached trie node from memory. If it cannot be found +// node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. -func (db *Database) Node(hash common.Hash) ([]byte, error) { +func (db *Database) node(hash common.Hash) ([]byte, error) { // It doesn't make sense to retrieve the metaroot if hash == (common.Hash{}) { return nil, errors.New("not found") @@ -198,11 +193,14 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return enc, nil } } - // Retrieve the node from the dirty cache if available + // Retrieve the node from the dirty cache if available. db.lock.RLock() dirty := db.dirties[hash] db.lock.RUnlock() + // Return the cached node if it's found in the dirty set. + // The dirty.node field is immutable and safe to read it + // even without lock guard. if dirty != nil { memcacheDirtyHitMeter.Mark(1) memcacheDirtyReadMeter.Mark(int64(len(dirty.node))) @@ -223,20 +221,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return nil, errors.New("not found") } -// Nodes retrieves the hashes of all the nodes cached within the memory database. -// This method is extremely expensive and should only be used to validate internal -// states in test code. -func (db *Database) Nodes() []common.Hash { - db.lock.RLock() - defer db.lock.RUnlock() - - var hashes = make([]common.Hash, 0, len(db.dirties)) - for hash := range db.dirties { - hashes = append(hashes, hash) - } - return hashes -} - // Reference adds a new reference from a parent node to a child node. // This function is used to add reference between internal trie node // and external node(e.g. storage trie root), all internal trie nodes @@ -344,16 +328,16 @@ func (db *Database) dereference(hash common.Hash) { // Cap iteratively flushes old but still referenced trie nodes until the total // memory usage goes below the given threshold. -// -// Note, this method is a non-synchronized mutator. It is unsafe to call this -// concurrently with other mutators. func (db *Database) Cap(limit common.StorageSize) error { + db.lock.Lock() + defer db.lock.Unlock() + // Create a database batch to flush persistent data out. It is important that // outside code doesn't see an inconsistent state (referenced data removed from // memory cache during commit but not yet in persistent storage). This is ensured // by only uncaching existing data when the database write finalizes. - nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() batch := db.diskdb.NewBatch() + nodes, storage, start := len(db.dirties), db.dirtiesSize, time.Now() // db.dirtiesSize only contains the useful data in the cache, but when reporting // the total memory consumption, the maintenance metadata is also needed to be @@ -391,9 +375,6 @@ func (db *Database) Cap(limit common.StorageSize) error { return err } // Write successful, clear out the flushed data - db.lock.Lock() - defer db.lock.Unlock() - for db.oldest != oldest { node := db.dirties[db.oldest] delete(db.dirties, db.oldest) @@ -424,10 +405,10 @@ func (db *Database) Cap(limit common.StorageSize) error { // Commit iterates over all the children of a particular node, writes them out // to disk, forcefully tearing down all references in both directions. As a side // effect, all pre-images accumulated up to this point are also written. -// -// Note, this method is a non-synchronized mutator. It is unsafe to call this -// concurrently with other mutators. func (db *Database) Commit(node common.Hash, report bool) error { + db.lock.Lock() + defer db.lock.Unlock() + // Create a database batch to flush persistent data out. It is important that // outside code doesn't see an inconsistent state (referenced data removed from // memory cache during commit but not yet in persistent storage). This is ensured @@ -449,8 +430,6 @@ func (db *Database) Commit(node common.Hash, report bool) error { return err } // Uncache any leftovers in the last batch - db.lock.Lock() - defer db.lock.Unlock() if err := batch.Replay(uncacher); err != nil { return err } @@ -499,13 +478,11 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane if err := batch.Write(); err != nil { return err } - db.lock.Lock() err := batch.Replay(uncacher) - batch.Reset() - db.lock.Unlock() if err != nil { return err } + batch.Reset() } return nil } @@ -574,7 +551,7 @@ func (db *Database) Initialized(genesisRoot common.Hash) bool { func (db *Database) Update(root common.Hash, parent common.Hash, block uint64, nodes *trienode.MergedNodeSet, states *triestate.Set) error { // Ensure the parent state is present and signal a warning if not. if parent != types.EmptyRootHash { - if blob, _ := db.Node(parent); len(blob) == 0 { + if blob, _ := db.node(parent); len(blob) == 0 { log.Error("parent state is not present") } } @@ -655,7 +632,7 @@ func (db *Database) Scheme() string { // Reader retrieves a node reader belonging to the given state root. // An error will be returned if the requested state is not available. func (db *Database) Reader(root common.Hash) (*reader, error) { - if _, err := db.Node(root); err != nil { + if _, err := db.node(root); err != nil { return nil, fmt.Errorf("state %#x is not available, %v", root, err) } return &reader{db: db}, nil @@ -666,9 +643,9 @@ type reader struct { db *Database } -// Node retrieves the trie node with the given node hash. -// No error will be returned if the node is not found. +// Node retrieves the trie node with the given node hash. No error will be +// returned if the node is not found. func (reader *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) { - blob, _ := reader.db.Node(hash) + blob, _ := reader.db.node(hash) return blob, nil } From 72410c399d6529e338743a7ed3ff9f9d05f9daaf Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Thu, 30 Nov 2023 10:35:38 -0500 Subject: [PATCH 17/43] Fixed `BlockchainLogger.OnBlockEnd` called too much times (#19) The `processBlock` is already doing the work to call `OnBlockEnd`, no need to call it anymore afterwards. --- cmd/evm/internal/t8ntool/execution.go | 2 +- core/blockchain.go | 6 ------ core/vm/contracts_fuzz_test.go | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 532e36fd1e..1ba3c26a8c 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -187,7 +187,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil { evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig) - core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb) + core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb, nil) } var blobGasUsed uint64 diff --git a/core/blockchain.go b/core/blockchain.go index d66ca0de0a..c496f677a9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1870,9 +1870,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) // After merge we expect few side chains. Simply count // all blocks the CL gives us for GC processing time bc.gcproc += res.procTime - if bc.logger != nil { - bc.logger.OnBlockEnd(nil) - } return it.index, nil // Direct block insertion of a single block } switch res.status { @@ -1901,9 +1898,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) } - if bc.logger != nil { - bc.logger.OnBlockEnd(nil) - } } // Any blocks remaining here? The only ones we care about are the future ones diff --git a/core/vm/contracts_fuzz_test.go b/core/vm/contracts_fuzz_test.go index 87c1fff7cc..1e5cc80074 100644 --- a/core/vm/contracts_fuzz_test.go +++ b/core/vm/contracts_fuzz_test.go @@ -36,7 +36,7 @@ func FuzzPrecompiledContracts(f *testing.F) { return } inWant := string(input) - RunPrecompiledContract(p, input, gas) + RunPrecompiledContract(p, input, gas, nil) if inHave := string(input); inWant != inHave { t.Errorf("Precompiled %v modified input data", a) } From f2b509d8a1ae877e2edb03560e57236eb74816b7 Mon Sep 17 00:00:00 2001 From: BorkBorked <107079055+BorkBorked@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:51:50 +0100 Subject: [PATCH 18/43] accounts/abi/bind: fix typo (#28630) --- accounts/abi/bind/util_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index b34c5bc226..16110b5d27 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -119,9 +119,9 @@ func TestWaitDeployedCornerCases(t *testing.T) { defer cancel() backend.SendTransaction(ctx, tx) backend.Commit() - notContentCreation := errors.New("tx is not contract creation") - if _, err := bind.WaitDeployed(ctx, backend, tx); err.Error() != notContentCreation.Error() { - t.Errorf("error mismatch: want %q, got %q, ", notContentCreation, err) + notContractCreation := errors.New("tx is not contract creation") + if _, err := bind.WaitDeployed(ctx, backend, tx); err.Error() != notContractCreation.Error() { + t.Errorf("error mismatch: want %q, got %q, ", notContractCreation, err) } // Create a transaction that is not mined. From dd0d0a2522ccad6bdeab1e84ff577bd5826540ee Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 1 Dec 2023 13:28:20 +0100 Subject: [PATCH 19/43] slog: faster and less memory-consumption (#28621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These changes improves the performance of the non-coloured terminal formatting, _quite a lot_. ``` name old time/op new time/op delta TerminalHandler-8 10.2µs ±15% 5.4µs ± 9% -47.02% (p=0.008 n=5+5) name old alloc/op new alloc/op delta TerminalHandler-8 2.17kB ± 0% 0.40kB ± 0% -81.46% (p=0.008 n=5+5) name old allocs/op new allocs/op delta TerminalHandler-8 33.0 ± 0% 5.0 ± 0% -84.85% (p=0.008 n=5+5) ``` I tried to _somewhat_ organize the commits, but the it might still be a bit chaotic. Some core insights: - The function `terminalHandler.Handl` uses a mutex, and writes all output immediately to 'upstream'. Thus, it can reuse a scratch-buffer every time. - This buffer can be propagated internally, making all the internal formatters either write directly to it, - OR, make use of the `tmp := buf.AvailableBuffer()` in some cases, where a byte buffer "extra capacity" can be temporarily used. - The `slog` package uses `Attr` by value. It makes sense to minimize operating on them, since iterating / collecting into a new slice, iterating again etc causes copy-on-heap. Better to operate on them only once. - If we want to do padding, it's better to copy from a constant `space`-buffer than to invoke `bytes.Repeat` every single time. --- internal/testlog/testlog.go | 3 +- log/format.go | 358 ++++++++++++++++++++---------------- log/format_test.go | 8 +- log/handler.go | 6 +- log/logger_test.go | 121 ++++++++++++ 5 files changed, 332 insertions(+), 164 deletions(-) diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index 68b9fb19f8..a7899c8158 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -183,8 +183,7 @@ func (h *bufHandler) terminalFormat(r slog.Record) string { } for _, attr := range attrs { - rawVal := attr.Value.Any() - fmt.Fprintf(buf, " %s=%s", attr.Key, log.FormatLogfmtValue(rawVal, true)) + fmt.Fprintf(buf, " %s=%s", attr.Key, string(log.FormatSlogValue(attr.Value, true, nil))) } buf.WriteByte('\n') return buf.String() diff --git a/log/format.go b/log/format.go index 5cbbe3341e..a2bbcce9c0 100644 --- a/log/format.go +++ b/log/format.go @@ -15,12 +15,14 @@ import ( const ( timeFormat = "2006-01-02T15:04:05-0700" - termTimeFormat = "01-02|15:04:05.000" floatFormat = 'f' termMsgJust = 40 termCtxMaxPadding = 40 ) +// 40 spaces +var spaces = []byte(" ") + type Format interface { Format(r slog.Record) []byte } @@ -44,37 +46,47 @@ type TerminalStringer interface { TerminalString() string } -func (h *TerminalHandler) TerminalFormat(r slog.Record, usecolor bool) []byte { +func (h *TerminalHandler) TerminalFormat(buf []byte, r slog.Record, usecolor bool) []byte { msg := escapeMessage(r.Message) - var color = 0 + var color = "" if usecolor { switch r.Level { case LevelCrit: - color = 35 + color = "\x1b[35m" case slog.LevelError: - color = 31 + color = "\x1b[31m" case slog.LevelWarn: - color = 33 + color = "\x1b[33m" case slog.LevelInfo: - color = 32 + color = "\x1b[32m" case slog.LevelDebug: - color = 36 + color = "\x1b[36m" case LevelTrace: - color = 34 + color = "\x1b[34m" } } - - b := &bytes.Buffer{} - lvl := LevelAlignedString(r.Level) - if color > 0 { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %s ", color, lvl, r.Time.Format(termTimeFormat), msg) - } else { - fmt.Fprintf(b, "%s[%s] %s ", lvl, r.Time.Format(termTimeFormat), msg) + if buf == nil { + buf = make([]byte, 0, 30+termMsgJust) } + b := bytes.NewBuffer(buf) + + if color != "" { // Start color + b.WriteString(color) + b.WriteString(LevelAlignedString(r.Level)) + b.WriteString("\x1b[0m") + } else { + b.WriteString(LevelAlignedString(r.Level)) + } + b.WriteString("[") + writeTimeTermFormat(b, r.Time) + b.WriteString("] ") + b.WriteString(msg) + // try to justify the log output for short messages - length := utf8.RuneCountInString(msg) - if r.NumAttrs() > 0 && length < termMsgJust { - b.Write(bytes.Repeat([]byte{' '}, termMsgJust-length)) + //length := utf8.RuneCountInString(msg) + length := len(msg) + if (r.NumAttrs()+len(h.attrs)) > 0 && length < termMsgJust { + b.Write(spaces[:termMsgJust-length]) } // print the keys logfmt style h.logfmt(b, r, color) @@ -82,150 +94,139 @@ func (h *TerminalHandler) TerminalFormat(r slog.Record, usecolor bool) []byte { return b.Bytes() } -func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color int) { - attrs := []slog.Attr{} - r.Attrs(func(attr slog.Attr) bool { - attrs = append(attrs, attr) - return true - }) +func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) { + // tmp is a temporary buffer we use, until bytes.Buffer.AvailableBuffer() (1.21) + // can be used. + var tmp = make([]byte, 40) + writeAttr := func(attr slog.Attr, first, last bool) { + buf.WriteByte(' ') - attrs = append(h.attrs, attrs...) - - for i, attr := range attrs { - if i != 0 { - buf.WriteByte(' ') - } - - key := escapeString(attr.Key) - rawVal := attr.Value.Any() - val := FormatLogfmtValue(rawVal, true) - - // XXX: we should probably check that all of your key bytes aren't invalid - // TODO (jwasinger) above comment was from log15 code. what does it mean? check that key bytes are ascii characters? - padding := h.fieldPadding[key] - - length := utf8.RuneCountInString(val) - if padding < length && length <= termCtxMaxPadding { - padding = length - h.fieldPadding[key] = padding - } - if color > 0 { - fmt.Fprintf(buf, "\x1b[%dm%s\x1b[0m=", color, key) + if color != "" { + buf.WriteString(color) + //buf.Write(appendEscapeString(buf.AvailableBuffer(), attr.Key)) + buf.Write(appendEscapeString(tmp[:0], attr.Key)) + buf.WriteString("\x1b[0m=") } else { - buf.WriteString(key) + //buf.Write(appendEscapeString(buf.AvailableBuffer(), attr.Key)) + buf.Write(appendEscapeString(tmp[:0], attr.Key)) buf.WriteByte('=') } - buf.WriteString(val) - if i < r.NumAttrs()-1 && padding > length { - buf.Write(bytes.Repeat([]byte{' '}, padding-length)) + //val := FormatSlogValue(attr.Value, true, buf.AvailableBuffer()) + val := FormatSlogValue(attr.Value, true, tmp[:0]) + + padding := h.fieldPadding[attr.Key] + + length := utf8.RuneCount(val) + if padding < length && length <= termCtxMaxPadding { + padding = length + h.fieldPadding[attr.Key] = padding + } + buf.Write(val) + if !last && padding > length { + buf.Write(spaces[:padding-length]) } } + var n = 0 + var nAttrs = len(h.attrs) + r.NumAttrs() + for _, attr := range h.attrs { + writeAttr(attr, n == 0, n == nAttrs-1) + n++ + } + r.Attrs(func(attr slog.Attr) bool { + writeAttr(attr, n == 0, n == nAttrs-1) + n++ + return true + }) buf.WriteByte('\n') } -// formatValue formats a value for serialization -func FormatLogfmtValue(value interface{}, term bool) (result string) { - if value == nil { - return "" - } +// FormatSlogValue formats a slog.Value for serialization +func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { + var value any defer func() { if err := recover(); err != nil { if v := reflect.ValueOf(value); v.Kind() == reflect.Ptr && v.IsNil() { - result = "" + result = []byte("") } else { panic(err) } } }() - switch v := value.(type) { - case time.Time: + switch v.Kind() { + case slog.KindString: + return appendEscapeString(tmp, v.String()) + case slog.KindAny: + value = v.Any() + case slog.KindInt64: // All int-types (int8 ,int16 etc) wind up here + return appendInt64(tmp, v.Int64()) + case slog.KindUint64: // All uint-types (int8 ,int16 etc) wind up here + return appendUint64(tmp, v.Uint64(), false) + case slog.KindFloat64: + return strconv.AppendFloat(tmp, v.Float64(), floatFormat, 3, 64) + case slog.KindBool: + return strconv.AppendBool(tmp, v.Bool()) + case slog.KindDuration: + value = v.Duration() + case slog.KindTime: // Performance optimization: No need for escaping since the provided // timeFormat doesn't have any escape characters, and escaping is // expensive. - return v.Format(timeFormat) - + return v.Time().AppendFormat(tmp, timeFormat) + default: + value = v.Any() + } + if value == nil { + return []byte("") + } + switch v := value.(type) { case *big.Int: // Big ints get consumed by the Stringer clause, so we need to handle // them earlier on. if v == nil { - return "" + return append(tmp, []byte("")...) } - return formatLogfmtBigInt(v) + return appendBigInt(tmp, v) case *uint256.Int: // Uint256s get consumed by the Stringer clause, so we need to handle // them earlier on. if v == nil { - return "" + return append(tmp, []byte("")...) } - return formatLogfmtUint256(v) - } - if term { - if s, ok := value.(TerminalStringer); ok { - // Custom terminal stringer provided, use that - return escapeString(s.TerminalString()) - } - } - switch v := value.(type) { + return appendU256(tmp, v) case error: - return escapeString(v.Error()) + return appendEscapeString(tmp, v.Error()) + case TerminalStringer: + if term { + return appendEscapeString(tmp, v.TerminalString()) // Custom terminal stringer provided, use that + } case fmt.Stringer: - return escapeString(v.String()) - case bool: - return strconv.FormatBool(v) - case float32: - return strconv.FormatFloat(float64(v), floatFormat, 3, 64) - case float64: - return strconv.FormatFloat(v, floatFormat, 3, 64) - case int8: - return strconv.FormatInt(int64(v), 10) - case uint8: - return strconv.FormatInt(int64(v), 10) - case int16: - return strconv.FormatInt(int64(v), 10) - case uint16: - return strconv.FormatInt(int64(v), 10) - // Larger integers get thousands separators. - case int: - return FormatLogfmtInt64(int64(v)) - case int32: - return FormatLogfmtInt64(int64(v)) - case int64: - return FormatLogfmtInt64(v) - case uint: - return FormatLogfmtUint64(uint64(v)) - case uint32: - return FormatLogfmtUint64(uint64(v)) - case uint64: - return FormatLogfmtUint64(v) - case string: - return escapeString(v) - default: - return escapeString(fmt.Sprintf("%+v", value)) + return appendEscapeString(tmp, v.String()) } + + // We can use the 'tmp' as a scratch-buffer, to first format the + // value, and in a second step do escaping. + internal := fmt.Appendf(tmp, "%+v", value) + return appendEscapeString(tmp, string(internal)) } -// FormatLogfmtInt64 formats n with thousand separators. -func FormatLogfmtInt64(n int64) string { +// appendInt64 formats n with thousand separators and writes into buffer dst. +func appendInt64(dst []byte, n int64) []byte { if n < 0 { - return formatLogfmtUint64(uint64(-n), true) + return appendUint64(dst, uint64(-n), true) } - return formatLogfmtUint64(uint64(n), false) + return appendUint64(dst, uint64(n), false) } -// FormatLogfmtUint64 formats n with thousand separators. -func FormatLogfmtUint64(n uint64) string { - return formatLogfmtUint64(n, false) -} - -func formatLogfmtUint64(n uint64, neg bool) string { +// appendUint64 formats n with thousand separators and writes into buffer dst. +func appendUint64(dst []byte, n uint64, neg bool) []byte { // Small numbers are fine as is if n < 100000 { if neg { - return strconv.Itoa(-int(n)) + return strconv.AppendInt(dst, -int64(n), 10) } else { - return strconv.Itoa(int(n)) + return strconv.AppendInt(dst, int64(n), 10) } } // Large numbers should be split @@ -250,16 +251,21 @@ func formatLogfmtUint64(n uint64, neg bool) string { out[i] = '-' i-- } - return string(out[i+1:]) + return append(dst, out[i+1:]...) } -// formatLogfmtBigInt formats n with thousand separators. -func formatLogfmtBigInt(n *big.Int) string { +// FormatLogfmtUint64 formats n with thousand separators. +func FormatLogfmtUint64(n uint64) string { + return string(appendUint64(nil, n, false)) +} + +// appendBigInt formats n with thousand separators and writes to dst. +func appendBigInt(dst []byte, n *big.Int) []byte { if n.IsUint64() { - return FormatLogfmtUint64(n.Uint64()) + return appendUint64(dst, n.Uint64(), false) } if n.IsInt64() { - return FormatLogfmtInt64(n.Int64()) + return appendInt64(dst, n.Int64()) } var ( @@ -284,54 +290,48 @@ func formatLogfmtBigInt(n *big.Int) string { comma++ } } - return string(buf[i+1:]) + return append(dst, buf[i+1:]...) } -// formatLogfmtUint256 formats n with thousand separators. -func formatLogfmtUint256(n *uint256.Int) string { +// appendU256 formats n with thousand separators. +func appendU256(dst []byte, n *uint256.Int) []byte { if n.IsUint64() { - return FormatLogfmtUint64(n.Uint64()) + return appendUint64(dst, n.Uint64(), false) } - var ( - text = n.Dec() - buf = make([]byte, len(text)+len(text)/3) - comma = 0 - i = len(buf) - 1 - ) - for j := len(text) - 1; j >= 0; j, i = j-1, i-1 { - c := text[j] - - switch { - case c == '-': - buf[i] = c - case comma == 3: - buf[i] = ',' - i-- - comma = 0 - fallthrough - default: - buf[i] = c - comma++ - } - } - return string(buf[i+1:]) + res := []byte(n.PrettyDec(',')) + return append(dst, res...) } -// escapeString checks if the provided string needs escaping/quoting, and -// calls strconv.Quote if needed -func escapeString(s string) string { +// appendEscapeString writes the string s to the given writer, with +// escaping/quoting if needed. +func appendEscapeString(dst []byte, s string) []byte { needsQuoting := false + needsEscaping := false for _, r := range s { - // We quote everything below " (0x22) and above~ (0x7E), plus equal-sign - if r <= '"' || r > '~' || r == '=' { + // If it contains spaces or equal-sign, we need to quote it. + if r == ' ' || r == '=' { needsQuoting = true + continue + } + // We need to escape it, if it contains + // - character " (0x22) and lower (except space) + // - characters above ~ (0x7E), plus equal-sign + if r <= '"' || r > '~' { + needsEscaping = true break } } - if !needsQuoting { - return s + if needsEscaping { + return strconv.AppendQuote(dst, s) } - return strconv.Quote(s) + // No escaping needed, but we might have to place within quote-marks, in case + // it contained a space + if needsQuoting { + dst = append(dst, '"') + dst = append(dst, []byte(s)...) + return append(dst, '"') + } + return append(dst, []byte(s)...) } // escapeMessage checks if the provided string needs escaping/quoting, similarly @@ -356,3 +356,45 @@ func escapeMessage(s string) string { } return strconv.Quote(s) } + +// writeTimeTermFormat writes on the format "01-02|15:04:05.000" +func writeTimeTermFormat(buf *bytes.Buffer, t time.Time) { + _, month, day := t.Date() + writePosIntWidth(buf, int(month), 2) + buf.WriteByte('-') + writePosIntWidth(buf, day, 2) + buf.WriteByte('|') + hour, min, sec := t.Clock() + writePosIntWidth(buf, hour, 2) + buf.WriteByte(':') + writePosIntWidth(buf, min, 2) + buf.WriteByte(':') + writePosIntWidth(buf, sec, 2) + ns := t.Nanosecond() + buf.WriteByte('.') + writePosIntWidth(buf, ns/1e6, 3) +} + +// writePosIntWidth writes non-negative integer i to the buffer, padded on the left +// by zeroes to the given width. Use a width of 0 to omit padding. +// Adapted from golang.org/x/exp/slog/internal/buffer/buffer.go +func writePosIntWidth(b *bytes.Buffer, i, width int) { + // Cheap integer to fixed-width decimal ASCII. + // Copied from log/log.go. + if i < 0 { + panic("negative int") + } + // Assemble decimal in reverse order. + var bb [20]byte + bp := len(bb) - 1 + for i >= 10 || width > 1 { + width-- + q := i / 10 + bb[bp] = byte('0' + i - q*10) + bp-- + i = q + } + // i < 10 + bb[bp] = byte('0' + i) + b.Write(bb[bp:]) +} diff --git a/log/format_test.go b/log/format_test.go index 41e1809c38..d4c1df4abc 100644 --- a/log/format_test.go +++ b/log/format_test.go @@ -5,18 +5,20 @@ import ( "testing" ) -var sink string +var sink []byte func BenchmarkPrettyInt64Logfmt(b *testing.B) { + buf := make([]byte, 100) b.ReportAllocs() for i := 0; i < b.N; i++ { - sink = FormatLogfmtInt64(rand.Int63()) + sink = appendInt64(buf, rand.Int63()) } } func BenchmarkPrettyUint64Logfmt(b *testing.B) { + buf := make([]byte, 100) b.ReportAllocs() for i := 0; i < b.N; i++ { - sink = FormatLogfmtUint64(rand.Uint64()) + sink = appendUint64(buf, rand.Uint64(), false) } } diff --git a/log/handler.go b/log/handler.go index ef1bcc1992..1a25577450 100644 --- a/log/handler.go +++ b/log/handler.go @@ -81,6 +81,8 @@ type TerminalHandler struct { // fieldPadding is a map with maximum field value lengths seen until now // to allow padding log contexts in a bit smarter way. fieldPadding map[string]int + + buf []byte } // NewTerminalHandler returns a handler which formats log records at all levels optimized for human readability on @@ -110,7 +112,9 @@ func NewTerminalHandlerWithLevel(wr io.Writer, lvl slog.Level, useColor bool) *T func (h *TerminalHandler) Handle(_ context.Context, r slog.Record) error { h.mu.Lock() defer h.mu.Unlock() - h.wr.Write(h.TerminalFormat(r, h.useColor)) + buf := h.TerminalFormat(h.buf, r, h.useColor) + h.wr.Write(buf) + h.buf = buf[:0] return nil } diff --git a/log/logger_test.go b/log/logger_test.go index fca1f1680f..27e90c5fd2 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -2,10 +2,15 @@ package log import ( "bytes" + "fmt" + "io" + "math/big" "os" "strings" "testing" + "time" + "github.com/holiman/uint256" "golang.org/x/exp/slog" ) @@ -51,3 +56,119 @@ func BenchmarkTraceLogging(b *testing.B) { Trace("a message", "v", i) } } + +func BenchmarkTerminalHandler(b *testing.B) { + l := NewLogger(NewTerminalHandler(io.Discard, false)) + benchmarkLogger(b, l) +} +func BenchmarkLogfmtHandler(b *testing.B) { + l := NewLogger(LogfmtHandler(io.Discard)) + benchmarkLogger(b, l) +} + +func BenchmarkJSONHandler(b *testing.B) { + l := NewLogger(JSONHandler(io.Discard)) + benchmarkLogger(b, l) +} + +func benchmarkLogger(b *testing.B, l Logger) { + var ( + bb = make([]byte, 10) + tt = time.Now() + bigint = big.NewInt(100) + nilbig *big.Int + err = fmt.Errorf("Oh nooes it's crap") + ) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Info("This is a message", + "foo", int16(i), + "bytes", bb, + "bonk", "a string with text", + "time", tt, + "bigint", bigint, + "nilbig", nilbig, + "err", err) + } + b.StopTimer() +} + +func TestLoggerOutput(t *testing.T) { + type custom struct { + A string + B int8 + } + var ( + customA = custom{"Foo", 12} + customB = custom{"Foo\nLinebreak", 122} + bb = make([]byte, 10) + tt = time.Time{} + bigint = big.NewInt(100) + nilbig *big.Int + err = fmt.Errorf("Oh nooes it's crap") + lazy = Lazy{Fn: func() interface{} { return "lazy value" }} + smallUint = uint256.NewInt(500_000) + bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} + ) + + out := new(bytes.Buffer) + glogHandler := NewGlogHandler(NewTerminalHandler(out, false)) + glogHandler.Verbosity(LevelInfo) + NewLogger(glogHandler).Info("This is a message", + "foo", int16(123), + "bytes", bb, + "bonk", "a string with text", + "time", tt, + "bigint", bigint, + "nilbig", nilbig, + "err", err, + "struct", customA, + "struct", customB, + "ptrstruct", &customA, + "lazy", lazy, + "smalluint", smallUint, + "bigUint", bigUint) + + have := out.String() + t.Logf("output %v", out.String()) + want := `INFO [11-07|19:14:33.821] This is a message foo=123 bytes="[0 0 0 0 0 0 0 0 0 0]" bonk="a string with text" time=0001-01-01T00:00:00+0000 bigint=100 nilbig= err="Oh nooes it's crap" struct="{A:Foo B:12}" struct="{A:Foo\nLinebreak B:122}" ptrstruct="&{A:Foo B:12}" lazy="lazy value" smalluint=500,000 bigUint=1,600,660,942,523,603,594,864,898,306,482,794,244,293,965,082,972,225,630,372,095 +` + if !bytes.Equal([]byte(have)[25:], []byte(want)[25:]) { + t.Errorf("Error\nhave: %q\nwant: %q", have, want) + } +} + +const termTimeFormat = "01-02|15:04:05.000" + +func BenchmarkAppendFormat(b *testing.B) { + var now = time.Now() + b.Run("fmt time.Format", func(b *testing.B) { + for i := 0; i < b.N; i++ { + fmt.Fprintf(io.Discard, "%s", now.Format(termTimeFormat)) + } + }) + b.Run("time.AppendFormat", func(b *testing.B) { + for i := 0; i < b.N; i++ { + now.AppendFormat(nil, termTimeFormat) + } + }) + var buf = new(bytes.Buffer) + b.Run("time.Custom", func(b *testing.B) { + for i := 0; i < b.N; i++ { + writeTimeTermFormat(buf, now) + buf.Reset() + } + }) +} + +func TestTermTimeFormat(t *testing.T) { + var now = time.Now() + want := now.AppendFormat(nil, termTimeFormat) + var b = new(bytes.Buffer) + writeTimeTermFormat(b, now) + have := b.Bytes() + if !bytes.Equal(have, want) { + t.Errorf("have != want\nhave: %q\nwant: %q\n", have, want) + } +} From 5fb8ebc9ecb226b84181420b9871c5f61cf4f77d Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 1 Dec 2023 21:08:52 +0800 Subject: [PATCH 20/43] eth/tracers: tx-level state in debug_traceCall (#28460) --- eth/tracers/api.go | 18 +++++-- eth/tracers/api_test.go | 116 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 300d904a99..7c0028601d 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -164,6 +164,7 @@ type TraceCallConfig struct { TraceConfig StateOverrides *ethapi.StateOverride BlockOverrides *ethapi.BlockOverrides + TxIndex *hexutil.Uint } // StdTraceConfig holds extra parameters to standard-json trace functions. @@ -863,11 +864,17 @@ func (api *API) TraceTransaction(ctx context.Context, hash common.Hash, config * // TraceCall lets you trace a given eth_call. It collects the structured logs // created during the execution of EVM if the given transaction was added on // top of the provided block and returns them as a JSON object. +// If no transaction index is specified, the trace will be conducted on the state +// after executing the specified block. However, if a transaction index is provided, +// the trace will be conducted on the state after executing the specified transaction +// within the specified block. func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, config *TraceCallConfig) (interface{}, error) { // Try to retrieve the specified block var ( - err error - block *types.Block + err error + block *types.Block + statedb *state.StateDB + release StateReleaseFunc ) if hash, ok := blockNrOrHash.Hash(); ok { block, err = api.blockByHash(ctx, hash) @@ -892,7 +899,12 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc if config != nil && config.Reexec != nil { reexec = *config.Reexec } - statedb, release, err := api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) + + if config != nil && config.TxIndex != nil { + _, _, statedb, release, err = api.backend.StateAtTransaction(ctx, block, int(*config.TxIndex), reexec) + } else { + statedb, release, err = api.backend.StateAtBlock(ctx, block, reexec, nil, true, false) + } if err != nil { return nil, err } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 0f78af9a01..49c3ebb67d 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -200,13 +200,51 @@ func TestTraceCall(t *testing.T) { } genBlocks := 10 signer := types.HomesteadSigner{} + nonce := uint64(0) backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei - tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) b.AddTx(tx) + nonce++ + + if i == genBlocks-2 { + // Transfer from account[0] to account[2] + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &accounts[2].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + nonce++ + + // Transfer from account[0] to account[1] again + tx, _ = types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) + b.AddTx(tx) + nonce++ + } }) + + uintPtr := func(i int) *hexutil.Uint { x := hexutil.Uint(i); return &x } + defer backend.teardown() api := NewAPI(backend) var testSuite = []struct { @@ -240,6 +278,51 @@ func TestTraceCall(t *testing.T) { expectErr: nil, expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, }, + // Upon the last state, default to the post block's state + { + blockNumber: rpc.BlockNumber(genBlocks - 1), + call: ethapi.TransactionArgs{ + From: &accounts[2].addr, + To: &accounts[0].addr, + Value: (*hexutil.Big)(new(big.Int).Add(big.NewInt(params.Ether), big.NewInt(100))), + }, + config: nil, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, + }, + // Before the first transaction, should be failed + { + blockNumber: rpc.BlockNumber(genBlocks - 1), + call: ethapi.TransactionArgs{ + From: &accounts[2].addr, + To: &accounts[0].addr, + Value: (*hexutil.Big)(new(big.Int).Add(big.NewInt(params.Ether), big.NewInt(100))), + }, + config: &TraceCallConfig{TxIndex: uintPtr(0)}, + expectErr: fmt.Errorf("tracing failed: insufficient funds for gas * price + value: address %s have 1000000000000000000 want 1000000000000000100", accounts[2].addr), + }, + // Before the target transaction, should be failed + { + blockNumber: rpc.BlockNumber(genBlocks - 1), + call: ethapi.TransactionArgs{ + From: &accounts[2].addr, + To: &accounts[0].addr, + Value: (*hexutil.Big)(new(big.Int).Add(big.NewInt(params.Ether), big.NewInt(100))), + }, + config: &TraceCallConfig{TxIndex: uintPtr(1)}, + expectErr: fmt.Errorf("tracing failed: insufficient funds for gas * price + value: address %s have 1000000000000000000 want 1000000000000000100", accounts[2].addr), + }, + // After the target transaction, should be succeed + { + blockNumber: rpc.BlockNumber(genBlocks - 1), + call: ethapi.TransactionArgs{ + From: &accounts[2].addr, + To: &accounts[0].addr, + Value: (*hexutil.Big)(new(big.Int).Add(big.NewInt(params.Ether), big.NewInt(100))), + }, + config: &TraceCallConfig{TxIndex: uintPtr(2)}, + expectErr: nil, + expect: `{"gas":21000,"failed":false,"returnValue":"","structLogs":[]}`, + }, // Standard JSON trace upon the non-existent block, error expects { blockNumber: rpc.BlockNumber(genBlocks + 1), @@ -297,8 +380,8 @@ func TestTraceCall(t *testing.T) { t.Errorf("test %d: expect error %v, got nothing", i, testspec.expectErr) continue } - if !reflect.DeepEqual(err, testspec.expectErr) { - t.Errorf("test %d: error mismatch, want %v, git %v", i, testspec.expectErr, err) + if !reflect.DeepEqual(err.Error(), testspec.expectErr.Error()) { + t.Errorf("test %d: error mismatch, want '%v', got '%v'", i, testspec.expectErr, err) } } else { if err != nil { @@ -338,7 +421,14 @@ func TestTraceTransaction(t *testing.T) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei - tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: uint64(i), + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) b.AddTx(tx) target = tx.Hash() }) @@ -388,7 +478,14 @@ func TestTraceBlock(t *testing.T) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei - tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: uint64(i), + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) b.AddTx(tx) txHash = tx.Hash() }) @@ -478,7 +575,14 @@ func TestTracingWithOverrides(t *testing.T) { // Transfer from account[0] to account[1] // value: 1000 wei // fee: 0 wei - tx, _ := types.SignTx(types.NewTransaction(uint64(i), accounts[1].addr, big.NewInt(1000), params.TxGas, b.BaseFee(), nil), signer, accounts[0].key) + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: uint64(i), + To: &accounts[1].addr, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, accounts[0].key) b.AddTx(tx) }) defer backend.chain.Stop() From 6e488c244934ea84062cb35d3af26670e1d20e25 Mon Sep 17 00:00:00 2001 From: ddl Date: Mon, 4 Dec 2023 17:52:55 +0800 Subject: [PATCH 21/43] cmd/evm: fix Env struct json tag (#28635) --- cmd/evm/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/evm/README.md b/cmd/evm/README.md index e6c6fe06ad..41d8ced278 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -88,7 +88,7 @@ type Env struct { CurrentTimestamp uint64 `json:"currentTimestamp"` Withdrawals []*Withdrawal `json:"withdrawals"` // optional - CurrentDifficulty *big.Int `json:"currentDifficuly"` + CurrentDifficulty *big.Int `json:"currentDifficulty"` CurrentRandom *big.Int `json:"currentRandom"` CurrentBaseFee *big.Int `json:"currentBaseFee"` ParentDifficulty *big.Int `json:"parentDifficulty"` From f04e5bde7487ce554930187e766164b18c37d867 Mon Sep 17 00:00:00 2001 From: BorkBorked <107079055+BorkBorked@users.noreply.github.com> Date: Mon, 4 Dec 2023 10:53:42 +0100 Subject: [PATCH 22/43] accounts/abi/bind: fixed typos (#28634) * Update auth.go * Update backend.go * Update bind.go * Update bind_test.go --- accounts/abi/bind/auth.go | 2 +- accounts/abi/bind/backend.go | 2 +- accounts/abi/bind/bind.go | 2 +- accounts/abi/bind/bind_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/accounts/abi/bind/auth.go b/accounts/abi/bind/auth.go index 494dc88a57..91913ec3b2 100644 --- a/accounts/abi/bind/auth.go +++ b/accounts/abi/bind/auth.go @@ -56,7 +56,7 @@ func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) { } // NewKeyStoreTransactor is a utility method to easily create a transaction signer from -// an decrypted key from a keystore. +// a decrypted key from a keystore. // // Deprecated: Use NewKeyStoreTransactorWithChainID instead. func NewKeyStoreTransactor(keystore *keystore.KeyStore, account accounts.Account) (*TransactOpts, error) { diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index d13b919641..2e45e86ae2 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -75,7 +75,7 @@ type BlockHashContractCaller interface { // CodeAtHash returns the code of the given account in the state at the specified block hash. CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error) - // CallContractAtHash executes an Ethereum contract all against the state at the specified block hash. + // CallContractAtHash executes an Ethereum contract call against the state at the specified block hash. CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error) } diff --git a/accounts/abi/bind/bind.go b/accounts/abi/bind/bind.go index 8a54a0e6ef..ec28013463 100644 --- a/accounts/abi/bind/bind.go +++ b/accounts/abi/bind/bind.go @@ -363,7 +363,7 @@ func bindTopicTypeGo(kind abi.Type, structs map[string]*tmplStruct) string { // parameters that are not value types i.e. arrays and structs are not // stored directly but instead a keccak256-hash of an encoding is stored. // - // We only convert stringS and bytes to hash, still need to deal with + // We only convert strings and bytes to hash, still need to deal with // array(both fixed-size and dynamic-size) and struct. if bound == "string" || bound == "[]byte" { bound = "common.Hash" diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 1069f3d396..3191167a00 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1677,7 +1677,7 @@ var bindTests = []struct { } sim.Commit() - // This test the existence of the free retreiver call for view and pure functions + // This test the existence of the free retriever call for view and pure functions if num, err := pav.PureFunc(nil); err != nil { t.Fatalf("Failed to call anonymous field retriever: %v", err) } else if num.Cmp(big.NewInt(42)) != 0 { From 70fd0b635e4198fe8695ff14f62c6736e71b5f27 Mon Sep 17 00:00:00 2001 From: Manav Darji Date: Mon, 4 Dec 2023 17:55:17 +0530 Subject: [PATCH 23/43] eth/fetcher: fix invalid tracking of received at time for block (#28637) eth/fetcher: fix invalid tracking of received at time --- eth/fetcher/block_fetcher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 8751c4e3ea..126eaaea7f 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -483,7 +483,7 @@ func (f *BlockFetcher) loop() { select { case res := <-resCh: res.Done <- nil - f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now().Add(res.Time)) + f.FilterHeaders(peer, *res.Res.(*eth.BlockHeadersRequest), time.Now()) case <-timeout.C: // The peer didn't respond in time. The request From 2e13b01046aade91e0fe2097a08b761acd413364 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5vard=20Anda=20Estensen?= Date: Mon, 4 Dec 2023 14:55:06 +0100 Subject: [PATCH 24/43] accounts: run tests in parallel (#28544) --- accounts/abi/abi_test.go | 24 ++++++++++++++++ accounts/abi/abifuzzer_test.go | 1 + accounts/abi/bind/backends/simulated_test.go | 29 ++++++++++++++++++++ accounts/abi/bind/base_test.go | 10 +++++++ accounts/abi/bind/bind_test.go | 1 + accounts/abi/bind/util_test.go | 2 ++ accounts/abi/event_test.go | 6 ++++ accounts/abi/method_test.go | 2 ++ accounts/abi/pack_test.go | 5 ++++ accounts/abi/reflect_test.go | 4 +++ accounts/abi/selector_parser_test.go | 1 + accounts/abi/topics_test.go | 9 ++++++ accounts/abi/type_test.go | 5 ++++ accounts/abi/unpack_test.go | 12 ++++++++ accounts/accounts_test.go | 1 + accounts/hd_test.go | 2 ++ accounts/keystore/account_cache_test.go | 3 ++ accounts/keystore/keystore_test.go | 7 +++++ accounts/keystore/passphrase_test.go | 1 + accounts/keystore/plain_test.go | 4 +++ accounts/url_test.go | 5 ++++ 21 files changed, 134 insertions(+) diff --git a/accounts/abi/abi_test.go b/accounts/abi/abi_test.go index 84175df4bb..bc76df0dc2 100644 --- a/accounts/abi/abi_test.go +++ b/accounts/abi/abi_test.go @@ -120,6 +120,7 @@ var methods = map[string]Method{ } func TestReader(t *testing.T) { + t.Parallel() abi := ABI{ Methods: methods, } @@ -151,6 +152,7 @@ func TestReader(t *testing.T) { } func TestInvalidABI(t *testing.T) { + t.Parallel() json := `[{ "type" : "function", "name" : "", "constant" : fals }]` _, err := JSON(strings.NewReader(json)) if err == nil { @@ -170,6 +172,7 @@ func TestInvalidABI(t *testing.T) { // constructor(uint256 a, uint256 b) public{} // } func TestConstructor(t *testing.T) { + t.Parallel() json := `[{ "inputs": [{"internalType": "uint256","name": "a","type": "uint256" },{ "internalType": "uint256","name": "b","type": "uint256"}],"stateMutability": "nonpayable","type": "constructor"}]` method := NewMethod("", "", Constructor, "nonpayable", false, false, []Argument{{"a", Uint256, false}, {"b", Uint256, false}}, nil) // Test from JSON @@ -199,6 +202,7 @@ func TestConstructor(t *testing.T) { } func TestTestNumbers(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -236,6 +240,7 @@ func TestTestNumbers(t *testing.T) { } func TestMethodSignature(t *testing.T) { + t.Parallel() m := NewMethod("foo", "foo", Function, "", false, false, []Argument{{"bar", String, false}, {"baz", String, false}}, nil) exp := "foo(string,string)" if m.Sig != exp { @@ -274,6 +279,7 @@ func TestMethodSignature(t *testing.T) { } func TestOverloadedMethodSignature(t *testing.T) { + t.Parallel() json := `[{"constant":true,"inputs":[{"name":"i","type":"uint256"},{"name":"j","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"constant":true,"inputs":[{"name":"i","type":"uint256"}],"name":"foo","outputs":[],"payable":false,"stateMutability":"pure","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"}],"name":"bar","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"i","type":"uint256"},{"indexed":false,"name":"j","type":"uint256"}],"name":"bar","type":"event"}]` abi, err := JSON(strings.NewReader(json)) if err != nil { @@ -297,6 +303,7 @@ func TestOverloadedMethodSignature(t *testing.T) { } func TestCustomErrors(t *testing.T) { + t.Parallel() json := `[{ "inputs": [ { "internalType": "uint256", "name": "", "type": "uint256" } ],"name": "MyError", "type": "error"} ]` abi, err := JSON(strings.NewReader(json)) if err != nil { @@ -311,6 +318,7 @@ func TestCustomErrors(t *testing.T) { } func TestMultiPack(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -348,6 +356,7 @@ func ExampleJSON() { } func TestInputVariableInputLength(t *testing.T) { + t.Parallel() const definition = `[ { "type" : "function", "name" : "strOne", "constant" : true, "inputs" : [ { "name" : "str", "type" : "string" } ] }, { "type" : "function", "name" : "bytesOne", "constant" : true, "inputs" : [ { "name" : "str", "type" : "bytes" } ] }, @@ -476,6 +485,7 @@ func TestInputVariableInputLength(t *testing.T) { } func TestInputFixedArrayAndVariableInputLength(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Error(err) @@ -650,6 +660,7 @@ func TestInputFixedArrayAndVariableInputLength(t *testing.T) { } func TestDefaultFunctionParsing(t *testing.T) { + t.Parallel() const definition = `[{ "name" : "balance", "type" : "function" }]` abi, err := JSON(strings.NewReader(definition)) @@ -663,6 +674,7 @@ func TestDefaultFunctionParsing(t *testing.T) { } func TestBareEvents(t *testing.T) { + t.Parallel() const definition = `[ { "type" : "event", "name" : "balance" }, { "type" : "event", "name" : "anon", "anonymous" : true}, @@ -739,6 +751,7 @@ func TestBareEvents(t *testing.T) { // // receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestUnpackEvent(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -777,6 +790,7 @@ func TestUnpackEvent(t *testing.T) { } func TestUnpackEventIntoMap(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"},{"indexed":false,"name":"amount","type":"uint256"},{"indexed":false,"name":"memo","type":"bytes"}],"name":"received","type":"event"},{"anonymous":false,"inputs":[{"indexed":false,"name":"sender","type":"address"}],"name":"receivedAddr","type":"event"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -827,6 +841,7 @@ func TestUnpackEventIntoMap(t *testing.T) { } func TestUnpackMethodIntoMap(t *testing.T) { + t.Parallel() const abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"receive","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[],"name":"send","outputs":[{"name":"amount","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"}],"name":"get","outputs":[{"name":"hash","type":"bytes"}],"payable":true,"stateMutability":"payable","type":"function"}]` abi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -877,6 +892,7 @@ func TestUnpackMethodIntoMap(t *testing.T) { } func TestUnpackIntoMapNamingConflict(t *testing.T) { + t.Parallel() // Two methods have the same name var abiJSON = `[{"constant":false,"inputs":[{"name":"memo","type":"bytes"}],"name":"get","outputs":[],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[],"name":"send","outputs":[{"name":"amount","type":"uint256"}],"payable":true,"stateMutability":"payable","type":"function"},{"constant":false,"inputs":[{"name":"addr","type":"address"}],"name":"get","outputs":[{"name":"hash","type":"bytes"}],"payable":true,"stateMutability":"payable","type":"function"}]` abi, err := JSON(strings.NewReader(abiJSON)) @@ -960,6 +976,7 @@ func TestUnpackIntoMapNamingConflict(t *testing.T) { } func TestABI_MethodById(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -992,6 +1009,7 @@ func TestABI_MethodById(t *testing.T) { } func TestABI_EventById(t *testing.T) { + t.Parallel() tests := []struct { name string json string @@ -1058,6 +1076,7 @@ func TestABI_EventById(t *testing.T) { } func TestABI_ErrorByID(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(`[ {"inputs":[{"internalType":"uint256","name":"x","type":"uint256"}],"name":"MyError1","type":"error"}, {"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"string","name":"b","type":"string"},{"internalType":"address","name":"c","type":"address"}],"internalType":"struct MyError.MyStruct","name":"x","type":"tuple"},{"internalType":"address","name":"y","type":"address"},{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"string","name":"b","type":"string"},{"internalType":"address","name":"c","type":"address"}],"internalType":"struct MyError.MyStruct","name":"z","type":"tuple"}],"name":"MyError2","type":"error"}, @@ -1088,6 +1107,7 @@ func TestABI_ErrorByID(t *testing.T) { // TestDoubleDuplicateMethodNames checks that if transfer0 already exists, there won't be a name // conflict and that the second transfer method will be renamed transfer1. func TestDoubleDuplicateMethodNames(t *testing.T) { + t.Parallel() abiJSON := `[{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"}],"name":"transfer0","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"},{"constant":false,"inputs":[{"name":"to","type":"address"},{"name":"value","type":"uint256"},{"name":"data","type":"bytes"},{"name":"customFallback","type":"string"}],"name":"transfer","outputs":[{"name":"ok","type":"bool"}],"payable":false,"stateMutability":"nonpayable","type":"function"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1117,6 +1137,7 @@ func TestDoubleDuplicateMethodNames(t *testing.T) { // event send(); // } func TestDoubleDuplicateEventNames(t *testing.T) { + t.Parallel() abiJSON := `[{"anonymous": false,"inputs": [{"indexed": false,"internalType": "uint256","name": "a","type": "uint256"}],"name": "send","type": "event"},{"anonymous": false,"inputs": [],"name": "send0","type": "event"},{ "anonymous": false, "inputs": [],"name": "send","type": "event"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1144,6 +1165,7 @@ func TestDoubleDuplicateEventNames(t *testing.T) { // event send(uint256, uint256); // } func TestUnnamedEventParam(t *testing.T) { + t.Parallel() abiJSON := `[{ "anonymous": false, "inputs": [{ "indexed": false,"internalType": "uint256", "name": "","type": "uint256"},{"indexed": false,"internalType": "uint256","name": "","type": "uint256"}],"name": "send","type": "event"}]` contractAbi, err := JSON(strings.NewReader(abiJSON)) if err != nil { @@ -1177,7 +1199,9 @@ func TestUnpackRevert(t *testing.T) { {"4e487b7100000000000000000000000000000000000000000000000000000000000000ff", "unknown panic code: 0xff", nil}, } for index, c := range cases { + index, c := index, c t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) { + t.Parallel() got, err := UnpackRevert(common.Hex2Bytes(c.input)) if c.expectErr != nil { if err == nil { diff --git a/accounts/abi/abifuzzer_test.go b/accounts/abi/abifuzzer_test.go index 4b67947815..dbf6ab6c54 100644 --- a/accounts/abi/abifuzzer_test.go +++ b/accounts/abi/abifuzzer_test.go @@ -28,6 +28,7 @@ import ( // TestReplicate can be used to replicate crashers from the fuzzing tests. // Just replace testString with the data in .quoted func TestReplicate(t *testing.T) { + t.Parallel() //t.Skip("Test only useful for reproducing issues") fuzzAbi([]byte("\x20\x20\x20\x20\x20\x20\x20\x20\x80\x00\x00\x00\x20\x20\x20\x20\x00")) //fuzzAbi([]byte("asdfasdfkadsf;lasdf;lasd;lfk")) diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index a41d168411..a2acf7ead5 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -38,6 +38,7 @@ import ( ) func TestSimulatedBackend(t *testing.T) { + t.Parallel() var gasLimit uint64 = 8000029 key, _ := crypto.GenerateKey() // nolint: gosec auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) @@ -121,6 +122,7 @@ func simTestBackend(testAddr common.Address) *SimulatedBackend { } func TestNewSimulatedBackend(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) expectedBal := big.NewInt(10000000000000000) sim := simTestBackend(testAddr) @@ -142,6 +144,7 @@ func TestNewSimulatedBackend(t *testing.T) { } func TestAdjustTime(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -159,6 +162,7 @@ func TestAdjustTime(t *testing.T) { } func TestNewAdjustTimeFail(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.blockchain.Stop() @@ -202,6 +206,7 @@ func TestNewAdjustTimeFail(t *testing.T) { } func TestBalanceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) expectedBal := big.NewInt(10000000000000000) sim := simTestBackend(testAddr) @@ -219,6 +224,7 @@ func TestBalanceAt(t *testing.T) { } func TestBlockByHash(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -240,6 +246,7 @@ func TestBlockByHash(t *testing.T) { } func TestBlockByNumber(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, ) @@ -275,6 +282,7 @@ func TestBlockByNumber(t *testing.T) { } func TestNonceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -328,6 +336,7 @@ func TestNonceAt(t *testing.T) { } func TestSendTransaction(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -362,6 +371,7 @@ func TestSendTransaction(t *testing.T) { } func TestTransactionByHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := NewSimulatedBackend( @@ -416,6 +426,7 @@ func TestTransactionByHash(t *testing.T) { } func TestEstimateGas(t *testing.T) { + t.Parallel() /* pragma solidity ^0.6.4; contract GasEstimation { @@ -535,6 +546,7 @@ func TestEstimateGas(t *testing.T) { } func TestEstimateGasWithPrice(t *testing.T) { + t.Parallel() key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) @@ -625,6 +637,7 @@ func TestEstimateGasWithPrice(t *testing.T) { } func TestHeaderByHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -646,6 +659,7 @@ func TestHeaderByHash(t *testing.T) { } func TestHeaderByNumber(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -692,6 +706,7 @@ func TestHeaderByNumber(t *testing.T) { } func TestTransactionCount(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -744,6 +759,7 @@ func TestTransactionCount(t *testing.T) { } func TestTransactionInBlock(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -809,6 +825,7 @@ func TestTransactionInBlock(t *testing.T) { } func TestPendingNonceAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -874,6 +891,7 @@ func TestPendingNonceAt(t *testing.T) { } func TestTransactionReceipt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) @@ -908,6 +926,7 @@ func TestTransactionReceipt(t *testing.T) { } func TestSuggestGasPrice(t *testing.T) { + t.Parallel() sim := NewSimulatedBackend( core.GenesisAlloc{}, 10000000, @@ -924,6 +943,7 @@ func TestSuggestGasPrice(t *testing.T) { } func TestPendingCodeAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -960,6 +980,7 @@ func TestPendingCodeAt(t *testing.T) { } func TestCodeAt(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -997,6 +1018,7 @@ func TestCodeAt(t *testing.T) { } func TestCodeAtHash(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1037,6 +1059,7 @@ func TestCodeAtHash(t *testing.T) { // // receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} func TestPendingAndCallContract(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1138,6 +1161,7 @@ contract Reverter { } }*/ func TestCallContractRevert(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1233,6 +1257,7 @@ func TestCallContractRevert(t *testing.T) { // Since Commit() was called 2n+1 times in total, // having a chain length of just n+1 means that a reorg occurred. func TestFork(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1286,6 +1311,7 @@ const callableBin = "6080604052348015600f57600080fd5b5060998061001e6000396000f3f // 9. Re-send the transaction and mine a block. // 10. Check that the event was reborn. func TestForkLogsReborn(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1359,6 +1385,7 @@ func TestForkLogsReborn(t *testing.T) { // 5. Mine a block, Re-send the transaction and mine another one. // 6. Check that the TX is now included in block 2. func TestForkResendTx(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1395,6 +1422,7 @@ func TestForkResendTx(t *testing.T) { } func TestCommitReturnValue(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() @@ -1436,6 +1464,7 @@ func TestCommitReturnValue(t *testing.T) { // TestAdjustTimeAfterFork ensures that after a fork, AdjustTime uses the pending fork // block's parent rather than the canonical head's parent. func TestAdjustTimeAfterFork(t *testing.T) { + t.Parallel() testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) defer sim.Close() diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index 44552ab121..f7eb7d14d3 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -135,6 +135,7 @@ func (mc *mockBlockHashCaller) CallContractAtHash(ctx context.Context, call ethe } func TestPassingBlockNumber(t *testing.T) { + t.Parallel() mc := &mockPendingCaller{ mockCaller: &mockCaller{ codeAtBytes: []byte{1, 2, 3}, @@ -186,6 +187,7 @@ func TestPassingBlockNumber(t *testing.T) { const hexData = "0x000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158" func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) { + t.Parallel() hash := crypto.Keccak256Hash([]byte("testName")) topics := []common.Hash{ crypto.Keccak256Hash([]byte("received(string,address,uint256,bytes)")), @@ -207,6 +209,7 @@ func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) { } func TestUnpackAnonymousLogIntoMap(t *testing.T) { + t.Parallel() mockLog := newMockLog(nil, common.HexToHash("0x0")) abiString := `[{"anonymous":false,"inputs":[{"indexed":false,"name":"amount","type":"uint256"}],"name":"received","type":"event"}]` @@ -224,6 +227,7 @@ func TestUnpackAnonymousLogIntoMap(t *testing.T) { } func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) { + t.Parallel() sliceBytes, err := rlp.EncodeToBytes([]string{"name1", "name2", "name3", "name4"}) if err != nil { t.Fatal(err) @@ -249,6 +253,7 @@ func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedArrayTyLogIntoMap(t *testing.T) { + t.Parallel() arrBytes, err := rlp.EncodeToBytes([2]common.Address{common.HexToAddress("0x0"), common.HexToAddress("0x376c47978271565f56DEB45495afa69E59c16Ab2")}) if err != nil { t.Fatal(err) @@ -274,6 +279,7 @@ func TestUnpackIndexedArrayTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedFuncTyLogIntoMap(t *testing.T) { + t.Parallel() mockAddress := common.HexToAddress("0x376c47978271565f56DEB45495afa69E59c16Ab2") addrBytes := mockAddress.Bytes() hash := crypto.Keccak256Hash([]byte("mockFunction(address,uint)")) @@ -300,6 +306,7 @@ func TestUnpackIndexedFuncTyLogIntoMap(t *testing.T) { } func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) { + t.Parallel() bytes := []byte{1, 2, 3, 4, 5} hash := crypto.Keccak256Hash(bytes) topics := []common.Hash{ @@ -322,6 +329,7 @@ func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) { } func TestTransactGasFee(t *testing.T) { + t.Parallel() assert := assert.New(t) // GasTipCap and GasFeeCap @@ -397,6 +405,7 @@ func newMockLog(topics []common.Hash, txHash common.Hash) types.Log { } func TestCall(t *testing.T) { + t.Parallel() var method, methodWithArg = "something", "somethingArrrrg" tests := []struct { name, method string @@ -572,6 +581,7 @@ func TestCall(t *testing.T) { // TestCrashers contains some strings which previously caused the abi codec to crash. func TestCrashers(t *testing.T) { + t.Parallel() abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"_1"}]}]}]`)) abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"&"}]}]}]`)) abi.JSON(strings.NewReader(`[{"inputs":[{"type":"tuple[]","components":[{"type":"bool","name":"----"}]}]}]`)) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 3191167a00..a5f7afa73c 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -2067,6 +2067,7 @@ var bindTests = []struct { // Tests that packages generated by the binder can be successfully compiled and // the requested tester run against it. func TestGolangBindings(t *testing.T) { + t.Parallel() // Skip the test if no Go command can be found gocmd := runtime.GOROOT() + "/bin/go" if !common.FileExist(gocmd) { diff --git a/accounts/abi/bind/util_test.go b/accounts/abi/bind/util_test.go index 16110b5d27..826426632c 100644 --- a/accounts/abi/bind/util_test.go +++ b/accounts/abi/bind/util_test.go @@ -53,6 +53,7 @@ var waitDeployedTests = map[string]struct { } func TestWaitDeployed(t *testing.T) { + t.Parallel() for name, test := range waitDeployedTests { backend := backends.NewSimulatedBackend( core.GenesisAlloc{ @@ -100,6 +101,7 @@ func TestWaitDeployed(t *testing.T) { } func TestWaitDeployedCornerCases(t *testing.T) { + t.Parallel() backend := backends.NewSimulatedBackend( core.GenesisAlloc{ crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)}, diff --git a/accounts/abi/event_test.go b/accounts/abi/event_test.go index 8f73419496..fffe28ea63 100644 --- a/accounts/abi/event_test.go +++ b/accounts/abi/event_test.go @@ -81,6 +81,7 @@ var pledgeData1 = "00000000000000000000000000ce0d46d924cc8437c806721496599fc3ffa var mixedCaseData1 = "00000000000000000000000000000000000000000000000000000000000f42400000000000000000000000000000000000000000000000000000020489e8000000000000000000000000000000000000000000000000000000000000000f4241" func TestEventId(t *testing.T) { + t.Parallel() var table = []struct { definition string expectations map[string]common.Hash @@ -112,6 +113,7 @@ func TestEventId(t *testing.T) { } func TestEventString(t *testing.T) { + t.Parallel() var table = []struct { definition string expectations map[string]string @@ -146,6 +148,7 @@ func TestEventString(t *testing.T) { // TestEventMultiValueWithArrayUnpack verifies that array fields will be counted after parsing array. func TestEventMultiValueWithArrayUnpack(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": false, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` abi, err := JSON(strings.NewReader(definition)) require.NoError(t, err) @@ -161,6 +164,7 @@ func TestEventMultiValueWithArrayUnpack(t *testing.T) { } func TestEventTupleUnpack(t *testing.T) { + t.Parallel() type EventTransfer struct { Value *big.Int } @@ -351,6 +355,7 @@ func unpackTestEventData(dest interface{}, hexData string, jsonEvent []byte, ass // TestEventUnpackIndexed verifies that indexed field will be skipped by event decoder. func TestEventUnpackIndexed(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8"},{"indexed": false, "name":"value2", "type":"uint8"}]}]` type testStruct struct { Value1 uint8 // indexed @@ -368,6 +373,7 @@ func TestEventUnpackIndexed(t *testing.T) { // TestEventIndexedWithArrayUnpack verifies that decoder will not overflow when static array is indexed input. func TestEventIndexedWithArrayUnpack(t *testing.T) { + t.Parallel() definition := `[{"name": "test", "type": "event", "inputs": [{"indexed": true, "name":"value1", "type":"uint8[2]"},{"indexed": false, "name":"value2", "type":"string"}]}]` type testStruct struct { Value1 [2]uint8 // indexed diff --git a/accounts/abi/method_test.go b/accounts/abi/method_test.go index 9230e307aa..6322173920 100644 --- a/accounts/abi/method_test.go +++ b/accounts/abi/method_test.go @@ -35,6 +35,7 @@ const methoddata = ` ]` func TestMethodString(t *testing.T) { + t.Parallel() var table = []struct { method string expectation string @@ -99,6 +100,7 @@ func TestMethodString(t *testing.T) { } func TestMethodSig(t *testing.T) { + t.Parallel() var cases = []struct { method string expect string diff --git a/accounts/abi/pack_test.go b/accounts/abi/pack_test.go index 5c7cb1cc1a..00bdae469e 100644 --- a/accounts/abi/pack_test.go +++ b/accounts/abi/pack_test.go @@ -32,8 +32,11 @@ import ( // TestPack tests the general pack/unpack tests in packing_test.go func TestPack(t *testing.T) { + t.Parallel() for i, test := range packUnpackTests { + i, test := i, test t.Run(strconv.Itoa(i), func(t *testing.T) { + t.Parallel() encb, err := hex.DecodeString(test.packed) if err != nil { t.Fatalf("invalid hex %s: %v", test.packed, err) @@ -57,6 +60,7 @@ func TestPack(t *testing.T) { } func TestMethodPack(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(jsondata)) if err != nil { t.Fatal(err) @@ -177,6 +181,7 @@ func TestMethodPack(t *testing.T) { } func TestPackNumber(t *testing.T) { + t.Parallel() tests := []struct { value reflect.Value packed []byte diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index 76ef1ad2aa..6c7ae57087 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -170,8 +170,11 @@ var reflectTests = []reflectTest{ } func TestReflectNameToStruct(t *testing.T) { + t.Parallel() for _, test := range reflectTests { + test := test t.Run(test.name, func(t *testing.T) { + t.Parallel() m, err := mapArgNamesToStructFields(test.args, reflect.ValueOf(test.struc)) if len(test.err) > 0 { if err == nil || err.Error() != test.err { @@ -192,6 +195,7 @@ func TestReflectNameToStruct(t *testing.T) { } func TestConvertType(t *testing.T) { + t.Parallel() // Test Basic Struct type T struct { X *big.Int diff --git a/accounts/abi/selector_parser_test.go b/accounts/abi/selector_parser_test.go index f6f134492b..6cb0ae0e70 100644 --- a/accounts/abi/selector_parser_test.go +++ b/accounts/abi/selector_parser_test.go @@ -24,6 +24,7 @@ import ( ) func TestParseSelector(t *testing.T) { + t.Parallel() mkType := func(types ...interface{}) []ArgumentMarshaling { var result []ArgumentMarshaling for i, typeOrComponents := range types { diff --git a/accounts/abi/topics_test.go b/accounts/abi/topics_test.go index 30cf21d0b8..b31f58fba3 100644 --- a/accounts/abi/topics_test.go +++ b/accounts/abi/topics_test.go @@ -26,6 +26,7 @@ import ( ) func TestMakeTopics(t *testing.T) { + t.Parallel() type args struct { query [][]interface{} } @@ -117,7 +118,9 @@ func TestMakeTopics(t *testing.T) { }, } for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() got, err := MakeTopics(tt.args.query...) if (err != nil) != tt.wantErr { t.Errorf("makeTopics() error = %v, wantErr %v", err, tt.wantErr) @@ -347,10 +350,13 @@ func setupTopicsTests() []topicTest { } func TestParseTopics(t *testing.T) { + t.Parallel() tests := setupTopicsTests() for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() createObj := tt.args.createObj() if err := ParseTopics(createObj, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr { t.Errorf("parseTopics() error = %v, wantErr %v", err, tt.wantErr) @@ -364,10 +370,13 @@ func TestParseTopics(t *testing.T) { } func TestParseTopicsIntoMap(t *testing.T) { + t.Parallel() tests := setupTopicsTests() for _, tt := range tests { + tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() outMap := make(map[string]interface{}) if err := ParseTopicsIntoMap(outMap, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr { t.Errorf("parseTopicsIntoMap() error = %v, wantErr %v", err, tt.wantErr) diff --git a/accounts/abi/type_test.go b/accounts/abi/type_test.go index a72531ba27..ae69872ad8 100644 --- a/accounts/abi/type_test.go +++ b/accounts/abi/type_test.go @@ -31,6 +31,7 @@ type typeWithoutStringer Type // Tests that all allowed types get recognized by the type parser. func TestTypeRegexp(t *testing.T) { + t.Parallel() tests := []struct { blob string components []ArgumentMarshaling @@ -117,6 +118,7 @@ func TestTypeRegexp(t *testing.T) { } func TestTypeCheck(t *testing.T) { + t.Parallel() for i, test := range []struct { typ string components []ArgumentMarshaling @@ -308,6 +310,7 @@ func TestTypeCheck(t *testing.T) { } func TestInternalType(t *testing.T) { + t.Parallel() components := []ArgumentMarshaling{{Name: "a", Type: "int64"}} internalType := "struct a.b[]" kind := Type{ @@ -332,6 +335,7 @@ func TestInternalType(t *testing.T) { } func TestGetTypeSize(t *testing.T) { + t.Parallel() var testCases = []struct { typ string components []ArgumentMarshaling @@ -368,6 +372,7 @@ func TestGetTypeSize(t *testing.T) { } func TestNewFixedBytesOver32(t *testing.T) { + t.Parallel() _, err := NewType("bytes4096", "", nil) if err == nil { t.Errorf("fixed bytes with size over 32 is not spec'd") diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index a7ee1d9202..29891ec0a4 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -33,6 +33,7 @@ import ( // TestUnpack tests the general pack/unpack tests in packing_test.go func TestUnpack(t *testing.T) { + t.Parallel() for i, test := range packUnpackTests { t.Run(strconv.Itoa(i)+" "+test.def, func(t *testing.T) { //Unpack @@ -224,6 +225,7 @@ var unpackTests = []unpackTest{ // TestLocalUnpackTests runs test specially designed only for unpacking. // All test cases that can be used to test packing and unpacking should move to packing_test.go func TestLocalUnpackTests(t *testing.T) { + t.Parallel() for i, test := range unpackTests { t.Run(strconv.Itoa(i), func(t *testing.T) { //Unpack @@ -251,6 +253,7 @@ func TestLocalUnpackTests(t *testing.T) { } func TestUnpackIntoInterfaceSetDynamicArrayOutput(t *testing.T) { + t.Parallel() abi, err := JSON(strings.NewReader(`[{"constant":true,"inputs":[],"name":"testDynamicFixedBytes15","outputs":[{"name":"","type":"bytes15[]"}],"payable":false,"stateMutability":"view","type":"function"},{"constant":true,"inputs":[],"name":"testDynamicFixedBytes32","outputs":[{"name":"","type":"bytes32[]"}],"payable":false,"stateMutability":"view","type":"function"}]`)) if err != nil { t.Fatal(err) @@ -321,6 +324,7 @@ func methodMultiReturn(require *require.Assertions) (ABI, []byte, methodMultiOut } func TestMethodMultiReturn(t *testing.T) { + t.Parallel() type reversed struct { String string Int *big.Int @@ -400,6 +404,7 @@ func TestMethodMultiReturn(t *testing.T) { } func TestMultiReturnWithArray(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"type": "uint64[3]"}, {"type": "uint64"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -423,6 +428,7 @@ func TestMultiReturnWithArray(t *testing.T) { } func TestMultiReturnWithStringArray(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "uint256[3]"},{"name": "","type": "address"},{"name": "","type": "string[2]"},{"name": "","type": "bool"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -453,6 +459,7 @@ func TestMultiReturnWithStringArray(t *testing.T) { } func TestMultiReturnWithStringSlice(t *testing.T) { + t.Parallel() const definition = `[{"name" : "multi", "type": "function", "outputs": [{"name": "","type": "string[]"},{"name": "","type": "uint256[]"}]}]` abi, err := JSON(strings.NewReader(definition)) if err != nil { @@ -485,6 +492,7 @@ func TestMultiReturnWithStringSlice(t *testing.T) { } func TestMultiReturnWithDeeplyNestedArray(t *testing.T) { + t.Parallel() // Similar to TestMultiReturnWithArray, but with a special case in mind: // values of nested static arrays count towards the size as well, and any element following // after such nested array argument should be read with the correct offset, @@ -525,6 +533,7 @@ func TestMultiReturnWithDeeplyNestedArray(t *testing.T) { } func TestUnmarshal(t *testing.T) { + t.Parallel() const definition = `[ { "name" : "int", "type": "function", "outputs": [ { "type": "uint256" } ] }, { "name" : "bool", "type": "function", "outputs": [ { "type": "bool" } ] }, @@ -774,6 +783,7 @@ func TestUnmarshal(t *testing.T) { } func TestUnpackTuple(t *testing.T) { + t.Parallel() const simpleTuple = `[{"name":"tuple","type":"function","outputs":[{"type":"tuple","name":"ret","components":[{"type":"int256","name":"a"},{"type":"int256","name":"b"}]}]}]` abi, err := JSON(strings.NewReader(simpleTuple)) if err != nil { @@ -876,6 +886,7 @@ func TestUnpackTuple(t *testing.T) { } func TestOOMMaliciousInput(t *testing.T) { + t.Parallel() oomTests := []unpackTest{ { def: `[{"type": "uint8[]"}]`, @@ -946,6 +957,7 @@ func TestOOMMaliciousInput(t *testing.T) { } func TestPackAndUnpackIncompatibleNumber(t *testing.T) { + t.Parallel() var encodeABI Arguments uint256Ty, err := NewType("uint256", "", nil) if err != nil { diff --git a/accounts/accounts_test.go b/accounts/accounts_test.go index e8274f9f04..2c4138aa78 100644 --- a/accounts/accounts_test.go +++ b/accounts/accounts_test.go @@ -24,6 +24,7 @@ import ( ) func TestTextHash(t *testing.T) { + t.Parallel() hash := TextHash([]byte("Hello Joe")) want := hexutil.MustDecode("0xa080337ae51c4e064c189e113edd0ba391df9206e2f49db658bb32cf2911730b") if !bytes.Equal(hash, want) { diff --git a/accounts/hd_test.go b/accounts/hd_test.go index 0743bbe666..118ec5187b 100644 --- a/accounts/hd_test.go +++ b/accounts/hd_test.go @@ -25,6 +25,7 @@ import ( // Tests that HD derivation paths can be correctly parsed into our internal binary // representation. func TestHDPathParsing(t *testing.T) { + t.Parallel() tests := []struct { input string output DerivationPath @@ -89,6 +90,7 @@ func testDerive(t *testing.T, next func() DerivationPath, expected []string) { } func TestHdPathIteration(t *testing.T) { + t.Parallel() testDerive(t, DefaultIterator(DefaultBaseDerivationPath), []string{ "m/44'/60'/0'/0/0", "m/44'/60'/0'/0/1", diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 371d274441..48a238048f 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -152,6 +152,7 @@ func TestWatchNoDir(t *testing.T) { } func TestCacheInitialReload(t *testing.T) { + t.Parallel() cache, _ := newAccountCache(cachetestDir) accounts := cache.accounts() if !reflect.DeepEqual(accounts, cachetestAccounts) { @@ -160,6 +161,7 @@ func TestCacheInitialReload(t *testing.T) { } func TestCacheAddDeleteOrder(t *testing.T) { + t.Parallel() cache, _ := newAccountCache("testdata/no-such-dir") cache.watcher.running = true // prevent unexpected reloads @@ -244,6 +246,7 @@ func TestCacheAddDeleteOrder(t *testing.T) { } func TestCacheFind(t *testing.T) { + t.Parallel() dir := filepath.Join("testdata", "dir") cache, _ := newAccountCache(dir) cache.watcher.running = true // prevent unexpected reloads diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index deb7cae9f9..c9a23eddd6 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -36,6 +36,7 @@ import ( var testSigData = make([]byte, 32) func TestKeyStore(t *testing.T) { + t.Parallel() dir, ks := tmpKeyStore(t, true) a, err := ks.NewAccount("foo") @@ -70,6 +71,7 @@ func TestKeyStore(t *testing.T) { } func TestSign(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) pass := "" // not used but required by API @@ -86,6 +88,7 @@ func TestSign(t *testing.T) { } func TestSignWithPassphrase(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) pass := "passwd" @@ -280,6 +283,7 @@ type walletEvent struct { // Tests that wallet notifications and correctly fired when accounts are added // or deleted from the keystore. func TestWalletNotifications(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, false) // Subscribe to the wallet feed and collect events. @@ -341,6 +345,7 @@ func TestWalletNotifications(t *testing.T) { // TestImportExport tests the import functionality of a keystore. func TestImportECDSA(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) key, err := crypto.GenerateKey() if err != nil { @@ -359,6 +364,7 @@ func TestImportECDSA(t *testing.T) { // TestImportECDSA tests the import and export functionality of a keystore. func TestImportExport(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) acc, err := ks.NewAccount("old") if err != nil { @@ -387,6 +393,7 @@ func TestImportExport(t *testing.T) { // TestImportRace tests the keystore on races. // This test should fail under -race if importing races. func TestImportRace(t *testing.T) { + t.Parallel() _, ks := tmpKeyStore(t, true) acc, err := ks.NewAccount("old") if err != nil { diff --git a/accounts/keystore/passphrase_test.go b/accounts/keystore/passphrase_test.go index 1de43a96da..20ec0f5519 100644 --- a/accounts/keystore/passphrase_test.go +++ b/accounts/keystore/passphrase_test.go @@ -30,6 +30,7 @@ const ( // Tests that a json key file can be decrypted and encrypted in multiple rounds. func TestKeyEncryptDecrypt(t *testing.T) { + t.Parallel() keyjson, err := os.ReadFile("testdata/very-light-scrypt.json") if err != nil { t.Fatal(err) diff --git a/accounts/keystore/plain_test.go b/accounts/keystore/plain_test.go index 93165d5cd3..737eb7fd61 100644 --- a/accounts/keystore/plain_test.go +++ b/accounts/keystore/plain_test.go @@ -40,6 +40,7 @@ func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) { } func TestKeyStorePlain(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, false) pass := "" // not used but required by API @@ -60,6 +61,7 @@ func TestKeyStorePlain(t *testing.T) { } func TestKeyStorePassphrase(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, true) pass := "foo" @@ -80,6 +82,7 @@ func TestKeyStorePassphrase(t *testing.T) { } func TestKeyStorePassphraseDecryptionFail(t *testing.T) { + t.Parallel() _, ks := tmpKeyStoreIface(t, true) pass := "foo" @@ -93,6 +96,7 @@ func TestKeyStorePassphraseDecryptionFail(t *testing.T) { } func TestImportPreSaleKey(t *testing.T) { + t.Parallel() dir, ks := tmpKeyStoreIface(t, true) // file content of a presale key file generated with: diff --git a/accounts/url_test.go b/accounts/url_test.go index 52be4c558d..f481a1016d 100644 --- a/accounts/url_test.go +++ b/accounts/url_test.go @@ -21,6 +21,7 @@ import ( ) func TestURLParsing(t *testing.T) { + t.Parallel() url, err := parseURL("https://ethereum.org") if err != nil { t.Errorf("unexpected error: %v", err) @@ -40,6 +41,7 @@ func TestURLParsing(t *testing.T) { } func TestURLString(t *testing.T) { + t.Parallel() url := URL{Scheme: "https", Path: "ethereum.org"} if url.String() != "https://ethereum.org" { t.Errorf("expected: %v, got: %v", "https://ethereum.org", url.String()) @@ -52,6 +54,7 @@ func TestURLString(t *testing.T) { } func TestURLMarshalJSON(t *testing.T) { + t.Parallel() url := URL{Scheme: "https", Path: "ethereum.org"} json, err := url.MarshalJSON() if err != nil { @@ -63,6 +66,7 @@ func TestURLMarshalJSON(t *testing.T) { } func TestURLUnmarshalJSON(t *testing.T) { + t.Parallel() url := &URL{} err := url.UnmarshalJSON([]byte("\"https://ethereum.org\"")) if err != nil { @@ -77,6 +81,7 @@ func TestURLUnmarshalJSON(t *testing.T) { } func TestURLComparison(t *testing.T) { + t.Parallel() tests := []struct { urlA URL urlB URL From 3dc071e0367505f4234ef880b853115c4040cfad Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Dec 2023 09:37:48 +0100 Subject: [PATCH 25/43] eth/tracers/logger: make structlog/json-log stack hex again (#28628) * common/hexutil: define hex wrappers for uint256.Int * eth/tracers/logger: make structlog/json-log stack hex again * common/hexutil: goimports --- common/hexutil/json.go | 45 ++++++++++++++++++++++ common/hexutil/json_test.go | 60 +++++++++++++++++++++++++++++ eth/tracers/logger/gen_structlog.go | 16 ++++++-- eth/tracers/logger/logger.go | 1 + 4 files changed, 118 insertions(+), 4 deletions(-) diff --git a/common/hexutil/json.go b/common/hexutil/json.go index 50db208118..e0ac98f52d 100644 --- a/common/hexutil/json.go +++ b/common/hexutil/json.go @@ -23,6 +23,8 @@ import ( "math/big" "reflect" "strconv" + + "github.com/holiman/uint256" ) var ( @@ -30,6 +32,7 @@ var ( bigT = reflect.TypeOf((*Big)(nil)) uintT = reflect.TypeOf(Uint(0)) uint64T = reflect.TypeOf(Uint64(0)) + u256T = reflect.TypeOf((*uint256.Int)(nil)) ) // Bytes marshals/unmarshals as a JSON string with 0x prefix. @@ -225,6 +228,48 @@ func (b *Big) UnmarshalGraphQL(input interface{}) error { return err } +// U256 marshals/unmarshals as a JSON string with 0x prefix. +// The zero value marshals as "0x0". +type U256 uint256.Int + +// MarshalText implements encoding.TextMarshaler +func (b U256) MarshalText() ([]byte, error) { + u256 := (*uint256.Int)(&b) + return []byte(u256.Hex()), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (b *U256) UnmarshalJSON(input []byte) error { + // The uint256.Int.UnmarshalJSON method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + if !isString(input) { + return errNonString(u256T) + } + // The hex decoder needs to accept empty string ("") as '0', which uint256.Int + // would reject. + if len(input) == 2 { + (*uint256.Int)(b).Clear() + return nil + } + err := (*uint256.Int)(b).SetFromHex(string(input[1 : len(input)-1])) + if err != nil { + return &json.UnmarshalTypeError{Value: err.Error(), Type: u256T} + } + return nil +} + +// UnmarshalText implements encoding.TextUnmarshaler +func (b *U256) UnmarshalText(input []byte) error { + // The uint256.Int.UnmarshalText method accepts "dec", "0xhex"; we must be + // more strict, hence we check string and invoke SetFromHex directly. + return (*uint256.Int)(b).SetFromHex(string(input)) +} + +// String returns the hex encoding of b. +func (b *U256) String() string { + return (*uint256.Int)(b).Hex() +} + // Uint64 marshals/unmarshals as a JSON string with 0x prefix. // The zero value marshals as "0x0". type Uint64 uint64 diff --git a/common/hexutil/json_test.go b/common/hexutil/json_test.go index ed7d6fad1a..7cca300951 100644 --- a/common/hexutil/json_test.go +++ b/common/hexutil/json_test.go @@ -23,6 +23,8 @@ import ( "errors" "math/big" "testing" + + "github.com/holiman/uint256" ) func checkError(t *testing.T, input string, got, want error) bool { @@ -176,6 +178,64 @@ func TestUnmarshalBig(t *testing.T) { } } +var unmarshalU256Tests = []unmarshalTest{ + // invalid encoding + {input: "", wantErr: errJSONEOF}, + {input: "null", wantErr: errNonString(u256T)}, + {input: "10", wantErr: errNonString(u256T)}, + {input: `"0"`, wantErr: wrapTypeError(ErrMissingPrefix, u256T)}, + {input: `"0x"`, wantErr: wrapTypeError(ErrEmptyNumber, u256T)}, + {input: `"0x01"`, wantErr: wrapTypeError(ErrLeadingZero, u256T)}, + {input: `"0xx"`, wantErr: wrapTypeError(ErrSyntax, u256T)}, + {input: `"0x1zz01"`, wantErr: wrapTypeError(ErrSyntax, u256T)}, + { + input: `"0x10000000000000000000000000000000000000000000000000000000000000000"`, + wantErr: wrapTypeError(ErrBig256Range, u256T), + }, + + // valid encoding + {input: `""`, want: big.NewInt(0)}, + {input: `"0x0"`, want: big.NewInt(0)}, + {input: `"0x2"`, want: big.NewInt(0x2)}, + {input: `"0x2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0X2F2"`, want: big.NewInt(0x2f2)}, + {input: `"0x1122aaff"`, want: big.NewInt(0x1122aaff)}, + {input: `"0xbBb"`, want: big.NewInt(0xbbb)}, + {input: `"0xfffffffff"`, want: big.NewInt(0xfffffffff)}, + { + input: `"0x112233445566778899aabbccddeeff"`, + want: referenceBig("112233445566778899aabbccddeeff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffff"), + }, + { + input: `"0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"`, + want: referenceBig("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), + }, +} + +func TestUnmarshalU256(t *testing.T) { + for _, test := range unmarshalU256Tests { + var v U256 + err := json.Unmarshal([]byte(test.input), &v) + if !checkError(t, test.input, err, test.wantErr) { + continue + } + if test.want == nil { + continue + } + want := new(uint256.Int) + want.SetFromBig(test.want.(*big.Int)) + have := (*uint256.Int)(&v) + if want.Cmp(have) != 0 { + t.Errorf("input %s: value mismatch: have %x, want %x", test.input, have, want) + continue + } + } +} + func BenchmarkUnmarshalBig(b *testing.B) { input := []byte(`"0x123456789abcdef123456789abcdef"`) for i := 0; i < b.N; i++ { diff --git a/eth/tracers/logger/gen_structlog.go b/eth/tracers/logger/gen_structlog.go index df06a9ee6b..b406cb3445 100644 --- a/eth/tracers/logger/gen_structlog.go +++ b/eth/tracers/logger/gen_structlog.go @@ -23,7 +23,7 @@ func (s StructLog) MarshalJSON() ([]byte, error) { GasCost math.HexOrDecimal64 `json:"gasCost"` Memory hexutil.Bytes `json:"memory,omitempty"` MemorySize int `json:"memSize"` - Stack []uint256.Int `json:"stack"` + Stack []hexutil.U256 `json:"stack"` ReturnData hexutil.Bytes `json:"returnData,omitempty"` Storage map[common.Hash]common.Hash `json:"-"` Depth int `json:"depth"` @@ -39,7 +39,12 @@ func (s StructLog) MarshalJSON() ([]byte, error) { enc.GasCost = math.HexOrDecimal64(s.GasCost) enc.Memory = s.Memory enc.MemorySize = s.MemorySize - enc.Stack = s.Stack + if s.Stack != nil { + enc.Stack = make([]hexutil.U256, len(s.Stack)) + for k, v := range s.Stack { + enc.Stack[k] = hexutil.U256(v) + } + } enc.ReturnData = s.ReturnData enc.Storage = s.Storage enc.Depth = s.Depth @@ -59,7 +64,7 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { GasCost *math.HexOrDecimal64 `json:"gasCost"` Memory *hexutil.Bytes `json:"memory,omitempty"` MemorySize *int `json:"memSize"` - Stack []uint256.Int `json:"stack"` + Stack []hexutil.U256 `json:"stack"` ReturnData *hexutil.Bytes `json:"returnData,omitempty"` Storage map[common.Hash]common.Hash `json:"-"` Depth *int `json:"depth"` @@ -89,7 +94,10 @@ func (s *StructLog) UnmarshalJSON(input []byte) error { s.MemorySize = *dec.MemorySize } if dec.Stack != nil { - s.Stack = dec.Stack + s.Stack = make([]uint256.Int, len(dec.Stack)) + for k, v := range dec.Stack { + s.Stack[k] = uint256.Int(v) + } } if dec.ReturnData != nil { s.ReturnData = *dec.ReturnData diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index 4c9b910a27..2b36f9f492 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -83,6 +83,7 @@ type structLogMarshaling struct { GasCost math.HexOrDecimal64 Memory hexutil.Bytes ReturnData hexutil.Bytes + Stack []hexutil.U256 OpName string `json:"opName"` // adds call to OpName() in MarshalJSON ErrorString string `json:"error,omitempty"` // adds call to ErrorString() in MarshalJSON } From 5190cfc5cb24493311fb5724361dd3c519fa66f8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 5 Dec 2023 14:16:25 +0330 Subject: [PATCH 26/43] rm extra galloc capturing --- core/genesis.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/genesis.go b/core/genesis.go index 81efbd314b..ed2f1eb503 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -158,12 +158,13 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { // specification will be flushed as well. func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, bcLogger BlockchainLogger) error { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) - statedb.SetLogger(bcLogger) if err != nil { return err } for addr, account := range *ga { if account.Balance != nil { + // This is not actually logged via tracer because OnGenesisBlock + // already captures the allocations. statedb.AddBalance(addr, account.Balance, state.BalanceChangeGenesisBalance) } statedb.SetCode(addr, account.Code) From b8d44ed98b9bc8320f1f64e82c13cf32918f1f6b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 5 Dec 2023 11:54:44 +0100 Subject: [PATCH 27/43] log: remove lazy, remove unused interfaces, unexport methods (#28622) This change - Removes interface `log.Format`, - Removes method `log.FormatFunc`, - unexports `TerminalHandler.TerminalFormat` formatting methods (renamed to `TerminalHandler.format`) - removes the notion of `log.Lazy` values The lazy handler was useful in the old log package, since it could defer the evaluation of costly attributes until later in the log pipeline: thus, if the logging was done at 'Trace', we could skip evaluation if logging only was set to 'Info'. With the move to slog, this way of deferring evaluation is no longer needed, since slog introduced 'Enabled': the caller can thus do the evaluate-or-not decision at the callsite, which is much more straight-forward than dealing with lazy reflect-based evaluation. Also, lazy evaluation would not work with 'native' slog, as in, these two statements would be evaluated differently: ```golang log.Info("foo", "my lazy", lazyObj) slog.Info("foo", "my lazy", lazyObj) ``` --- cmd/geth/logtestcmd_active.go | 10 ++-- cmd/geth/testdata/logging/logtest-json.txt | 3 +- cmd/geth/testdata/logging/logtest-logfmt.txt | 3 +- .../testdata/logging/logtest-terminal.txt | 5 +- internal/testlog/testlog.go | 6 +- log/format.go | 55 ++++--------------- log/handler.go | 39 +------------ log/logger.go | 28 +++------- log/logger_test.go | 4 +- p2p/discover/table.go | 7 ++- p2p/msgrate/msgrate.go | 5 +- 11 files changed, 49 insertions(+), 116 deletions(-) diff --git a/cmd/geth/logtestcmd_active.go b/cmd/geth/logtestcmd_active.go index 0ca4cc621d..5cce1ec6ab 100644 --- a/cmd/geth/logtestcmd_active.go +++ b/cmd/geth/logtestcmd_active.go @@ -43,6 +43,7 @@ This command is only meant for testing. type customQuotedStringer struct { } + func (c customQuotedStringer) String() string { return "output with 'quotes'" } @@ -80,8 +81,6 @@ func logTest(ctx *cli.Context) error { log.Info("uint64", "18,446,744,073,709,551,615", uint64(math.MaxUint64)) } { // Special characters - - log.Info("Special chars in value", "key", "special \r\n\t chars") log.Info("Special chars in key", "special \n\t chars", "value") @@ -103,9 +102,6 @@ func logTest(ctx *cli.Context) error { var c customQuotedStringer log.Info("a custom stringer that emits quoted text", "output", c) } - { // Lazy eval - log.Info("Lazy evaluation of value", "key", log.Lazy{Fn: func() interface{} { return "lazy value" }}) - } { // Multi-line message log.Info("A message with wonky \U0001F4A9 characters") log.Info("A multiline message \nINFO [10-18|14:11:31.106] with wonky characters \U0001F4A9") @@ -166,6 +162,10 @@ func logTest(ctx *cli.Context) error { { // Logging with 'reserved' keys log.Info("Using keys 't', 'lvl', 'time', 'level' and 'msg'", "t", "t", "time", "time", "lvl", "lvl", "level", "level", "msg", "msg") } + { // Logging with wrong attr-value pairs + log.Info("Odd pair (1 attr)", "key") + log.Info("Odd pair (3 attr)", "key", "value", "key2") + } return nil } diff --git a/cmd/geth/testdata/logging/logtest-json.txt b/cmd/geth/testdata/logging/logtest-json.txt index bdc1ae4de6..3bfe718660 100644 --- a/cmd/geth/testdata/logging/logtest-json.txt +++ b/cmd/geth/testdata/logging/logtest-json.txt @@ -21,7 +21,6 @@ {"t":"2023-11-22T15:42:00.408197+08:00","lvl":"info","msg":"an error message with quotes","error":"this is an 'error'"} {"t":"2023-11-22T15:42:00.408202+08:00","lvl":"info","msg":"Custom Stringer value","2562047h47m16.854s":"2562047h47m16.854s"} {"t":"2023-11-22T15:42:00.408208+08:00","lvl":"info","msg":"a custom stringer that emits quoted text","output":"output with 'quotes'"} -{"t":"2023-11-22T15:42:00.408215+08:00","lvl":"info","msg":"Lazy evaluation of value","key":"lazy value"} {"t":"2023-11-22T15:42:00.408219+08:00","lvl":"info","msg":"A message with wonky 💩 characters"} {"t":"2023-11-22T15:42:00.408222+08:00","lvl":"info","msg":"A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩"} {"t":"2023-11-22T15:42:00.408226+08:00","lvl":"info","msg":"A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above"} @@ -49,3 +48,5 @@ {"t":"2023-11-22T15:42:00.40835+08:00","lvl":"info","msg":"raw nil","res":null} {"t":"2023-11-22T15:42:00.408354+08:00","lvl":"info","msg":"(*uint64)(nil)","res":null} {"t":"2023-11-22T15:42:00.408361+08:00","lvl":"info","msg":"Using keys 't', 'lvl', 'time', 'level' and 'msg'","t":"t","time":"time","lvl":"lvl","level":"level","msg":"msg"} +{"t":"2023-11-29T15:13:00.195655931+01:00","lvl":"info","msg":"Odd pair (1 attr)","key":null,"LOG_ERROR":"Normalized odd number of arguments by adding nil"} +{"t":"2023-11-29T15:13:00.195681832+01:00","lvl":"info","msg":"Odd pair (3 attr)","key":"value","key2":null,"LOG_ERROR":"Normalized odd number of arguments by adding nil"} diff --git a/cmd/geth/testdata/logging/logtest-logfmt.txt b/cmd/geth/testdata/logging/logtest-logfmt.txt index 114569e467..f20d66635d 100644 --- a/cmd/geth/testdata/logging/logtest-logfmt.txt +++ b/cmd/geth/testdata/logging/logtest-logfmt.txt @@ -21,7 +21,6 @@ t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColor t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="an error message with quotes" error="this is an 'error'" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Custom Stringer value" 2562047h47m16.854s=2562047h47m16.854s t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="a custom stringer that emits quoted text" output="output with 'quotes'" -t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Lazy evaluation of value" key="lazy value" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A message with wonky 💩 characters" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above" @@ -49,3 +48,5 @@ t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=nil-custom-struct res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="raw nil" res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg=(*uint64)(nil) res= t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Using keys 't', 'lvl', 'time', 'level' and 'msg'" t=t time=time lvl=lvl level=level msg=msg +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Odd pair (1 attr)" key= LOG_ERROR="Normalized odd number of arguments by adding nil" +t=xxxx-xx-xxTxx:xx:xx+xxxx lvl=info msg="Odd pair (3 attr)" key=value key2= LOG_ERROR="Normalized odd number of arguments by adding nil" diff --git a/cmd/geth/testdata/logging/logtest-terminal.txt b/cmd/geth/testdata/logging/logtest-terminal.txt index 4da3f49d46..e3b562117c 100644 --- a/cmd/geth/testdata/logging/logtest-terminal.txt +++ b/cmd/geth/testdata/logging/logtest-terminal.txt @@ -21,8 +21,7 @@ INFO [xx-xx|xx:xx:xx.xxx] "\x1b[35mColored\x1b[0m[" "\x1b[35mColo INFO [xx-xx|xx:xx:xx.xxx] an error message with quotes error="this is an 'error'" INFO [xx-xx|xx:xx:xx.xxx] Custom Stringer value 2562047h47m16.854s=2562047h47m16.854s INFO [xx-xx|xx:xx:xx.xxx] a custom stringer that emits quoted text output="output with 'quotes'" -INFO [xx-xx|xx:xx:xx.xxx] Lazy evaluation of value key="lazy value" -INFO [xx-xx|xx:xx:xx.xxx] "A message with wonky 💩 characters" +INFO [xx-xx|xx:xx:xx.xxx] "A message with wonky 💩 characters" INFO [xx-xx|xx:xx:xx.xxx] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩" INFO [xx-xx|xx:xx:xx.xxx] A multiline message LALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above @@ -50,3 +49,5 @@ INFO [xx-xx|xx:xx:xx.xxx] nil-custom-struct res= INFO [xx-xx|xx:xx:xx.xxx] raw nil res= INFO [xx-xx|xx:xx:xx.xxx] (*uint64)(nil) res= INFO [xx-xx|xx:xx:xx.xxx] Using keys 't', 'lvl', 'time', 'level' and 'msg' t=t time=time lvl=lvl level=level msg=msg +INFO [xx-xx|xx:xx:xx.xxx] Odd pair (1 attr) key= LOG_ERROR="Normalized odd number of arguments by adding nil" +INFO [xx-xx|xx:xx:xx.xxx] Odd pair (3 attr) key=value key2= LOG_ERROR="Normalized odd number of arguments by adding nil" diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index a7899c8158..037b7ee9c1 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -100,6 +100,10 @@ func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { func (l *logger) Write(level slog.Level, msg string, ctx ...interface{}) {} +func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { + return l.l.Enabled(ctx, level) +} + func (l *logger) Trace(msg string, ctx ...interface{}) { l.t.Helper() l.mu.Lock() @@ -183,7 +187,7 @@ func (h *bufHandler) terminalFormat(r slog.Record) string { } for _, attr := range attrs { - fmt.Fprintf(buf, " %s=%s", attr.Key, string(log.FormatSlogValue(attr.Value, true, nil))) + fmt.Fprintf(buf, " %s=%s", attr.Key, string(log.FormatSlogValue(attr.Value, nil))) } buf.WriteByte('\n') return buf.String() diff --git a/log/format.go b/log/format.go index a2bbcce9c0..6447f3c1f1 100644 --- a/log/format.go +++ b/log/format.go @@ -23,22 +23,6 @@ const ( // 40 spaces var spaces = []byte(" ") -type Format interface { - Format(r slog.Record) []byte -} - -// FormatFunc returns a new Format object which uses -// the given function to perform record formatting. -func FormatFunc(f func(slog.Record) []byte) Format { - return formatFunc(f) -} - -type formatFunc func(slog.Record) []byte - -func (f formatFunc) Format(r slog.Record) []byte { - return f(r) -} - // TerminalStringer is an analogous interface to the stdlib stringer, allowing // own types to have custom shortened serialization formats when printed to the // screen. @@ -46,7 +30,7 @@ type TerminalStringer interface { TerminalString() string } -func (h *TerminalHandler) TerminalFormat(buf []byte, r slog.Record, usecolor bool) []byte { +func (h *TerminalHandler) format(buf []byte, r slog.Record, usecolor bool) []byte { msg := escapeMessage(r.Message) var color = "" if usecolor { @@ -88,13 +72,13 @@ func (h *TerminalHandler) TerminalFormat(buf []byte, r slog.Record, usecolor boo if (r.NumAttrs()+len(h.attrs)) > 0 && length < termMsgJust { b.Write(spaces[:termMsgJust-length]) } - // print the keys logfmt style - h.logfmt(b, r, color) + // print the attributes + h.formatAttributes(b, r, color) return b.Bytes() } -func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) { +func (h *TerminalHandler) formatAttributes(buf *bytes.Buffer, r slog.Record, color string) { // tmp is a temporary buffer we use, until bytes.Buffer.AvailableBuffer() (1.21) // can be used. var tmp = make([]byte, 40) @@ -112,7 +96,7 @@ func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) buf.WriteByte('=') } //val := FormatSlogValue(attr.Value, true, buf.AvailableBuffer()) - val := FormatSlogValue(attr.Value, true, tmp[:0]) + val := FormatSlogValue(attr.Value, tmp[:0]) padding := h.fieldPadding[attr.Key] @@ -140,8 +124,8 @@ func (h *TerminalHandler) logfmt(buf *bytes.Buffer, r slog.Record, color string) buf.WriteByte('\n') } -// FormatSlogValue formats a slog.Value for serialization -func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { +// FormatSlogValue formats a slog.Value for serialization to terminal. +func FormatSlogValue(v slog.Value, tmp []byte) (result []byte) { var value any defer func() { if err := recover(); err != nil { @@ -156,11 +140,9 @@ func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { switch v.Kind() { case slog.KindString: return appendEscapeString(tmp, v.String()) - case slog.KindAny: - value = v.Any() - case slog.KindInt64: // All int-types (int8 ,int16 etc) wind up here + case slog.KindInt64: // All int-types (int8, int16 etc) wind up here return appendInt64(tmp, v.Int64()) - case slog.KindUint64: // All uint-types (int8 ,int16 etc) wind up here + case slog.KindUint64: // All uint-types (uint8, uint16 etc) wind up here return appendUint64(tmp, v.Uint64(), false) case slog.KindFloat64: return strconv.AppendFloat(tmp, v.Float64(), floatFormat, 3, 64) @@ -180,27 +162,14 @@ func FormatSlogValue(v slog.Value, term bool, tmp []byte) (result []byte) { return []byte("") } switch v := value.(type) { - case *big.Int: - // Big ints get consumed by the Stringer clause, so we need to handle - // them earlier on. - if v == nil { - return append(tmp, []byte("")...) - } + case *big.Int: // Need to be before fmt.Stringer-clause return appendBigInt(tmp, v) - - case *uint256.Int: - // Uint256s get consumed by the Stringer clause, so we need to handle - // them earlier on. - if v == nil { - return append(tmp, []byte("")...) - } + case *uint256.Int: // Need to be before fmt.Stringer-clause return appendU256(tmp, v) case error: return appendEscapeString(tmp, v.Error()) case TerminalStringer: - if term { - return appendEscapeString(tmp, v.TerminalString()) // Custom terminal stringer provided, use that - } + return appendEscapeString(tmp, v.TerminalString()) case fmt.Stringer: return appendEscapeString(tmp, v.String()) } diff --git a/log/handler.go b/log/handler.go index 1a25577450..7459aad891 100644 --- a/log/handler.go +++ b/log/handler.go @@ -13,42 +13,6 @@ import ( "golang.org/x/exp/slog" ) -// Lazy allows you to defer calculation of a logged value that is expensive -// to compute until it is certain that it must be evaluated with the given filters. -// -// You may wrap any function which takes no arguments to Lazy. It may return any -// number of values of any type. -type Lazy struct { - Fn interface{} -} - -func evaluateLazy(lz Lazy) (interface{}, error) { - t := reflect.TypeOf(lz.Fn) - - if t.Kind() != reflect.Func { - return nil, fmt.Errorf("INVALID_LAZY, not func: %+v", lz.Fn) - } - - if t.NumIn() > 0 { - return nil, fmt.Errorf("INVALID_LAZY, func takes args: %+v", lz.Fn) - } - - if t.NumOut() == 0 { - return nil, fmt.Errorf("INVALID_LAZY, no func return val: %+v", lz.Fn) - } - - value := reflect.ValueOf(lz.Fn) - results := value.Call([]reflect.Value{}) - if len(results) == 1 { - return results[0].Interface(), nil - } - values := make([]interface{}, len(results)) - for i, v := range results { - values[i] = v.Interface() - } - return values, nil -} - type discardHandler struct{} // DiscardHandler returns a no-op handler @@ -112,7 +76,7 @@ func NewTerminalHandlerWithLevel(wr io.Writer, lvl slog.Level, useColor bool) *T func (h *TerminalHandler) Handle(_ context.Context, r slog.Record) error { h.mu.Lock() defer h.mu.Unlock() - buf := h.TerminalFormat(h.buf, r, h.useColor) + buf := h.format(h.buf, r, h.useColor) h.wr.Write(buf) h.buf = buf[:0] return nil @@ -149,6 +113,7 @@ func (l *leveler) Level() slog.Level { return l.minLevel } +// JSONHandler returns a handler which prints records in JSON format. func JSONHandler(wr io.Writer) slog.Handler { return slog.NewJSONHandler(wr, &slog.HandlerOptions{ ReplaceAttr: builtinReplaceJSON, diff --git a/log/logger.go b/log/logger.go index 3e227745ad..93d62f080b 100644 --- a/log/logger.go +++ b/log/logger.go @@ -134,6 +134,9 @@ type Logger interface { // Write logs a message at the specified level Write(level slog.Level, msg string, attrs ...any) + + // Enabled reports whether l emits log records at the given context and level. + Enabled(ctx context.Context, level slog.Level) bool } type logger struct { @@ -159,26 +162,6 @@ func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if len(attrs)%2 != 0 { attrs = append(attrs, nil, errorKey, "Normalized odd number of arguments by adding nil") } - - // evaluate lazy values - var hadErr bool - for i := 1; i < len(attrs); i += 2 { - lz, ok := attrs[i].(Lazy) - if ok { - v, err := evaluateLazy(lz) - if err != nil { - hadErr = true - attrs[i] = err - } else { - attrs[i] = v - } - } - } - - if hadErr { - attrs = append(attrs, errorKey, "bad lazy") - } - r := slog.NewRecord(time.Now(), level, msg, pcs[0]) r.Add(attrs...) l.inner.Handler().Handle(context.Background(), r) @@ -196,6 +179,11 @@ func (l *logger) New(ctx ...interface{}) Logger { return l.With(ctx...) } +// Enabled reports whether l emits log records at the given context and level. +func (l *logger) Enabled(ctx context.Context, level slog.Level) bool { + return l.inner.Enabled(ctx, level) +} + func (l *logger) Trace(msg string, ctx ...interface{}) { l.Write(LevelTrace, msg, ctx...) } diff --git a/log/logger_test.go b/log/logger_test.go index 27e90c5fd2..a633f5ad7a 100644 --- a/log/logger_test.go +++ b/log/logger_test.go @@ -107,7 +107,6 @@ func TestLoggerOutput(t *testing.T) { bigint = big.NewInt(100) nilbig *big.Int err = fmt.Errorf("Oh nooes it's crap") - lazy = Lazy{Fn: func() interface{} { return "lazy value" }} smallUint = uint256.NewInt(500_000) bigUint = &uint256.Int{0xff, 0xff, 0xff, 0xff} ) @@ -126,13 +125,12 @@ func TestLoggerOutput(t *testing.T) { "struct", customA, "struct", customB, "ptrstruct", &customA, - "lazy", lazy, "smalluint", smallUint, "bigUint", bigUint) have := out.String() t.Logf("output %v", out.String()) - want := `INFO [11-07|19:14:33.821] This is a message foo=123 bytes="[0 0 0 0 0 0 0 0 0 0]" bonk="a string with text" time=0001-01-01T00:00:00+0000 bigint=100 nilbig= err="Oh nooes it's crap" struct="{A:Foo B:12}" struct="{A:Foo\nLinebreak B:122}" ptrstruct="&{A:Foo B:12}" lazy="lazy value" smalluint=500,000 bigUint=1,600,660,942,523,603,594,864,898,306,482,794,244,293,965,082,972,225,630,372,095 + want := `INFO [11-07|19:14:33.821] This is a message foo=123 bytes="[0 0 0 0 0 0 0 0 0 0]" bonk="a string with text" time=0001-01-01T00:00:00+0000 bigint=100 nilbig= err="Oh nooes it's crap" struct="{A:Foo B:12}" struct="{A:Foo\nLinebreak B:122}" ptrstruct="&{A:Foo B:12}" smalluint=500,000 bigUint=1,600,660,942,523,603,594,864,898,306,482,794,244,293,965,082,972,225,630,372,095 ` if !bytes.Equal([]byte(have)[25:], []byte(want)[25:]) { t.Errorf("Error\nhave: %q\nwant: %q", have, want) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index f476d2079f..e6dafb0dca 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -23,6 +23,7 @@ package discover import ( + "context" crand "crypto/rand" "encoding/binary" "fmt" @@ -330,8 +331,10 @@ func (tab *Table) loadSeedNodes() { seeds = append(seeds, tab.nursery...) for i := range seeds { seed := seeds[i] - age := log.Lazy{Fn: func() interface{} { return time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) }} - tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) + if tab.log.Enabled(context.Background(), log.LevelTrace) { + age := time.Since(tab.db.LastPongReceived(seed.ID(), seed.IP())) + tab.log.Trace("Found seed node in database", "id", seed.ID(), "addr", seed.addr(), "age", age) + } tab.addSeenNode(seed) } } diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 4f08792242..de1a3177db 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -18,6 +18,7 @@ package msgrate import ( + "context" "errors" "math" "sort" @@ -410,7 +411,9 @@ func (t *Trackers) tune() { t.tuned = time.Now() t.log.Debug("Recalculated msgrate QoS values", "rtt", t.roundtrip, "confidence", t.confidence, "ttl", t.targetTimeout(), "next", t.tuned.Add(t.roundtrip)) - t.log.Trace("Debug dump of mean capacities", "caps", log.Lazy{Fn: t.meanCapacities}) + if t.log.Enabled(context.Background(), log.LevelTrace) { + t.log.Trace("Debug dump of mean capacities", "caps", t.meanCapacities()) + } } // detune reduces the tracker's confidence in order to make fresh measurements From 9e5c96fdff55b8f43df7ef80e35b9acd3d14ba3e Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 5 Dec 2023 16:31:16 +0330 Subject: [PATCH 28/43] add chainConfig to block events --- core/blockchain.go | 10 +++++----- core/genesis.go | 2 +- eth/tracers/live/printer.go | 5 +++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index c496f677a9..5bc5bbd68d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -192,9 +192,9 @@ type BlockchainLogger interface { state.StateLogger // OnBlockStart is called before executing `block`. // `td` is the total difficulty prior to `block`. - OnBlockStart(block *types.Block, td *big.Int, finalized *types.Header, safe *types.Header) + OnBlockStart(block *types.Block, td *big.Int, finalized *types.Header, safe *types.Header, chainConfig *params.ChainConfig) OnBlockEnd(err error) - OnGenesisBlock(genesis *types.Block, alloc GenesisAlloc) + OnGenesisBlock(genesis *types.Block, alloc GenesisAlloc, chainConfig *params.ChainConfig) OnBeaconBlockRootStart(root common.Hash) OnBeaconBlockRootEnd() } @@ -461,7 +461,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis return nil, fmt.Errorf("live blockchain tracer requires genesis alloc to be set") } - bc.logger.OnGenesisBlock(bc.genesisBlock, alloc) + bc.logger.OnGenesisBlock(bc.genesisBlock, alloc, bc.chainConfig) } } @@ -1804,7 +1804,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) } stats.processed++ if bc.logger != nil { - bc.logger.OnBlockStart(block, bc.GetTd(block.ParentHash(), block.NumberU64()-1), bc.CurrentFinalBlock(), bc.CurrentSafeBlock()) + bc.logger.OnBlockStart(block, bc.GetTd(block.ParentHash(), block.NumberU64()-1), bc.CurrentFinalBlock(), bc.CurrentSafeBlock(), bc.chainConfig) bc.logger.OnBlockEnd(nil) } @@ -1932,7 +1932,7 @@ type blockProcessingResult struct { func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, followupInterrupt *atomic.Bool, start time.Time, setHead bool) (blockEndErr error, _ bool, _ *blockProcessingResult) { if bc.logger != nil { td := bc.GetTd(block.ParentHash(), block.NumberU64()-1) - bc.logger.OnBlockStart(block, td, bc.CurrentFinalBlock(), bc.CurrentSafeBlock()) + bc.logger.OnBlockStart(block, td, bc.CurrentFinalBlock(), bc.CurrentSafeBlock(), bc.chainConfig) defer func() { bc.logger.OnBlockEnd(blockEndErr) }() diff --git a/core/genesis.go b/core/genesis.go index ed2f1eb503..cd62eebd95 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -540,7 +540,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database, bcLogger Bloc return nil, errors.New("can't start clique chain without signers") } if bcLogger != nil { - bcLogger.OnGenesisBlock(block, g.Alloc) + bcLogger.OnGenesisBlock(block, g.Alloc, config) } // All the checks has passed, flush the states derived from the genesis // specification as well as the specification itself into the provided diff --git a/eth/tracers/live/printer.go b/eth/tracers/live/printer.go index 62129d2939..58022b3903 100644 --- a/eth/tracers/live/printer.go +++ b/eth/tracers/live/printer.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers/directory" + "github.com/ethereum/go-ethereum/params" ) func init() { @@ -84,7 +85,7 @@ func (p *Printer) CaptureTxEnd(receipt *types.Receipt, err error) { fmt.Printf("CaptureTxEnd: receipt=%s\n", buf) } -func (p *Printer) OnBlockStart(b *types.Block, td *big.Int, finalized, safe *types.Header) { +func (p *Printer) OnBlockStart(b *types.Block, td *big.Int, finalized, safe *types.Header, _ *params.ChainConfig) { if finalized != nil && safe != nil { fmt.Printf("OnBlockStart: b=%v, td=%v, finalized=%v, safe=%v\n", b.NumberU64(), td, finalized.Number.Uint64(), safe.Number.Uint64()) } else { @@ -96,7 +97,7 @@ func (p *Printer) OnBlockEnd(err error) { fmt.Printf("OnBlockEnd: err=%v\n", err) } -func (p *Printer) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc) { +func (p *Printer) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc, _ *params.ChainConfig) { fmt.Printf("OnGenesisBlock: b=%v, allocLength=%d\n", b.NumberU64(), len(alloc)) } From f862531c250fed8e2912a9a1d7ccdc075377cf83 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 5 Dec 2023 16:49:03 +0330 Subject: [PATCH 29/43] rm chainConfig from OnGenesisBlock --- core/blockchain.go | 4 ++-- core/genesis.go | 2 +- eth/tracers/live/printer.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 5bc5bbd68d..e7ba251c8f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -194,7 +194,7 @@ type BlockchainLogger interface { // `td` is the total difficulty prior to `block`. OnBlockStart(block *types.Block, td *big.Int, finalized *types.Header, safe *types.Header, chainConfig *params.ChainConfig) OnBlockEnd(err error) - OnGenesisBlock(genesis *types.Block, alloc GenesisAlloc, chainConfig *params.ChainConfig) + OnGenesisBlock(genesis *types.Block, alloc GenesisAlloc) OnBeaconBlockRootStart(root common.Hash) OnBeaconBlockRootEnd() } @@ -461,7 +461,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis return nil, fmt.Errorf("live blockchain tracer requires genesis alloc to be set") } - bc.logger.OnGenesisBlock(bc.genesisBlock, alloc, bc.chainConfig) + bc.logger.OnGenesisBlock(bc.genesisBlock, alloc) } } diff --git a/core/genesis.go b/core/genesis.go index cd62eebd95..ed2f1eb503 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -540,7 +540,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database, bcLogger Bloc return nil, errors.New("can't start clique chain without signers") } if bcLogger != nil { - bcLogger.OnGenesisBlock(block, g.Alloc, config) + bcLogger.OnGenesisBlock(block, g.Alloc) } // All the checks has passed, flush the states derived from the genesis // specification as well as the specification itself into the provided diff --git a/eth/tracers/live/printer.go b/eth/tracers/live/printer.go index 58022b3903..422b009e29 100644 --- a/eth/tracers/live/printer.go +++ b/eth/tracers/live/printer.go @@ -97,7 +97,7 @@ func (p *Printer) OnBlockEnd(err error) { fmt.Printf("OnBlockEnd: err=%v\n", err) } -func (p *Printer) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc, _ *params.ChainConfig) { +func (p *Printer) OnGenesisBlock(b *types.Block, alloc core.GenesisAlloc) { fmt.Printf("OnGenesisBlock: b=%v, allocLength=%d\n", b.NumberU64(), len(alloc)) } From 69576df2544d9a6c59c5659b82a064edc9845874 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 5 Dec 2023 14:45:40 +0100 Subject: [PATCH 30/43] .github: use github actions to run 32-bit linux tests (#28549) use github actions to run 32-bit linux tests --- .github/workflows/go.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/go.yml diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000000..7924c521e8 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,23 @@ +name: i386 linux tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + build: + runs-on: self-hosted + steps: + - uses: actions/checkout@v2 + - name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.21.4 + - name: Run tests + run: go test ./... + env: + GOOS: linux + GOARCH: 386 From 9360cab0df77575584f5d0e589df26e53a84de66 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 7 Dec 2023 11:18:38 +0330 Subject: [PATCH 31/43] assign values for balance reasons --- core/state/metadata.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/core/state/metadata.go b/core/state/metadata.go index 9e78e6816f..db17b92a9e 100644 --- a/core/state/metadata.go +++ b/core/state/metadata.go @@ -21,19 +21,23 @@ package state type BalanceChangeReason byte const ( - BalanceChangeUnspecified BalanceChangeReason = iota - BalanceChangeRewardMineUncle - BalanceChangeRewardMineBlock - BalanceChangeDaoRefundContract - BalanceChangeDaoAdjustBalance - BalanceChangeTransfer - BalanceChangeGenesisBalance - BalanceChangeGasBuy - BalanceChangeRewardTransactionFee - BalanceChangeGasRefund - BalanceChangeTouchAccount - BalanceChangeSuicideRefund - BalanceChangeSuicideWithdraw - BalanceChangeBurn - BalanceChangeWithdrawal + BalanceChangeUnspecified BalanceChangeReason = 0 + BalanceChangeRewardMineUncle = 1 + BalanceChangeRewardMineBlock = 2 + BalanceChangeDaoRefundContract = 3 + BalanceChangeDaoAdjustBalance = 4 + BalanceChangeTransfer = 5 + BalanceChangeGenesisBalance = 6 + BalanceChangeGasBuy = 7 + BalanceChangeRewardTransactionFee = 8 + BalanceChangeGasRefund = 9 + BalanceChangeTouchAccount = 10 + // TODO: rename (debit, credit) + BalanceChangeSuicideRefund = 11 + BalanceChangeSuicideWithdraw = 12 + // TODO: add method on statedb to track burn without Add/Sub balance + // Or: Track via OnBurn + // 1559 burn, blob burn, withdraw to self via selfdestruct burn, receive ether after selfdestruct (at end of tx) + BalanceChangeBurn = 13 + BalanceChangeWithdrawal = 14 ) From 0967291487146bdf256a4f829ae84bbd9a9fd2ca Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 7 Dec 2023 12:00:54 +0330 Subject: [PATCH 32/43] Track burnt and selfdestruct withdraw bal changes --- core/state/metadata.go | 20 +++++--- core/state/statedb.go | 13 +++++- core/state_transition.go | 98 ++++++++++++++++++++++++++++++---------- 3 files changed, 99 insertions(+), 32 deletions(-) diff --git a/core/state/metadata.go b/core/state/metadata.go index db17b92a9e..90ea3d1742 100644 --- a/core/state/metadata.go +++ b/core/state/metadata.go @@ -33,11 +33,19 @@ const ( BalanceChangeGasRefund = 9 BalanceChangeTouchAccount = 10 // TODO: rename (debit, credit) - BalanceChangeSuicideRefund = 11 + // BalanceChangeSuicideRefund is added to the recipient as indicated by a selfdestructing account. + BalanceChangeSuicideRefund = 11 + // BalanceChangeSuicideWithdraw is deducted from a contract due to self-destruct. + // This can happen either at the point of self-destruction, or at the end of the tx + // if ether was sent to contract post-selfdestruct. BalanceChangeSuicideWithdraw = 12 - // TODO: add method on statedb to track burn without Add/Sub balance - // Or: Track via OnBurn - // 1559 burn, blob burn, withdraw to self via selfdestruct burn, receive ether after selfdestruct (at end of tx) - BalanceChangeBurn = 13 - BalanceChangeWithdrawal = 14 + // BalanceChangeBurn accounts for: + // - EIP-1559 burnt fees + // - ether that is sent to a self-destructed contract within the same tx (captured at end of tx) + // Note it doesn't account for a self-destruct which appoints same contract as recipient. + BalanceChangeBurn = 13 + // BalanceChangeBurnRefund is refunded to an account at the end of transaction based on + // gas usage from the estimated burn amount. + BalanceChangeBurnRefund = 14 + BalanceChangeWithdrawal = 15 ) diff --git a/core/state/statedb.go b/core/state/statedb.go index 0182fbb1dd..f951aa490b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -476,13 +476,18 @@ func (s *StateDB) SelfDestruct(addr common.Address) { if stateObject == nil { return } + var ( + prev = new(big.Int).Set(stateObject.Balance()) + n = new(big.Int) + ) s.journal.append(selfDestructChange{ account: &addr, prev: stateObject.selfDestructed, - prevbalance: new(big.Int).Set(stateObject.Balance()), + prevbalance: prev, }) + s.logger.OnBalanceChange(addr, prev, n, BalanceChangeSuicideWithdraw) stateObject.markSelfdestructed() - stateObject.data.Balance = new(big.Int) + stateObject.data.Balance = n } func (s *StateDB) Selfdestruct6780(addr common.Address) { @@ -870,6 +875,10 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { if obj.selfDestructed || (deleteEmptyObjects && obj.empty()) { obj.deleted = true + // If ether was sent to account post-selfdestruct it is burnt. + if bal := obj.Balance(); bal.Sign() != 0 { + s.logger.OnBalanceChange(obj.address, bal, new(big.Int), BalanceChangeBurn) + } // We need to maintain account deletions explicitly (will remain // set indefinitely). Note only the first occurred self-destruct // event is tracked. diff --git a/core/state_transition.go b/core/state_transition.go index a50447f3de..f4a68ec38f 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -233,44 +233,84 @@ func (st *StateTransition) to() common.Address { } func (st *StateTransition) buyGas() error { - mgval := new(big.Int).SetUint64(st.msg.GasLimit) - mgval = mgval.Mul(mgval, st.msg.GasPrice) - balanceCheck := new(big.Int).Set(mgval) if st.msg.GasFeeCap != nil { - balanceCheck.SetUint64(st.msg.GasLimit) - balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) - balanceCheck.Add(balanceCheck, st.msg.Value) - } - if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { - if blobGas := st.blobGasUsed(); blobGas > 0 { - // Check that the user has enough funds to cover blobGasUsed * tx.BlobGasFeeCap - blobBalanceCheck := new(big.Int).SetUint64(blobGas) - blobBalanceCheck.Mul(blobBalanceCheck, st.msg.BlobGasFeeCap) - balanceCheck.Add(balanceCheck, blobBalanceCheck) - // Pay for blobGasUsed * actual blob fee - blobFee := new(big.Int).SetUint64(blobGas) - blobFee.Mul(blobFee, st.evm.Context.BlobBaseFee) - mgval.Add(mgval, blobFee) + if err := st.buy1559Gas(); err != nil { + return err + } + } else { + if err := st.buyLegacyGas(); err != nil { + return err } } - if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { - return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) + if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { + if err := st.buyBlobGas(); err != nil { + return err + } } + if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err } - if st.evm.Config.Tracer != nil { st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, vm.GasChangeTxInitialBalance) } - st.gasRemaining += st.msg.GasLimit - st.initialGas = st.msg.GasLimit + + return nil +} + +func (st *StateTransition) buyLegacyGas() error { + mgval := new(big.Int).SetUint64(st.msg.GasLimit) + mgval = mgval.Mul(mgval, st.msg.GasPrice) + balanceCheck := new(big.Int).Set(mgval) + if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { + return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) + } st.state.SubBalance(st.msg.From, mgval, state.BalanceChangeGasBuy) return nil } +// buy1559Gas purchases gas according to EIP-1559 rules. See: +// https://eips.ethereum.org/EIPS/eip-1559 +func (st *StateTransition) buy1559Gas() error { + balanceCheck := new(big.Int).SetUint64(st.msg.GasLimit) + balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) + balanceCheck.Add(balanceCheck, st.msg.Value) + if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { + return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) + } + + burnt := new(big.Int).SetUint64(st.msg.GasLimit) + burnt = burnt.Mul(burnt, st.evm.Context.BaseFee) + st.state.SubBalance(st.msg.From, burnt, state.BalanceChangeBurn) + + effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) + tipVal := new(big.Int).SetUint64(st.msg.GasLimit) + tipVal = tipVal.Mul(tipVal, effectiveTip) + st.state.SubBalance(st.msg.From, tipVal, state.BalanceChangeGasBuy) + + return nil +} + +// buyBlobGas purchases blob gas as per: +// https://eips.ethereum.org/EIPS/eip-4844 +func (st *StateTransition) buyBlobGas() error { + if blobGas := st.blobGasUsed(); blobGas > 0 { + // Check that the user has enough funds to cover blobGasUsed * tx.BlobGasFeeCap + balanceCheck := new(big.Int).SetUint64(blobGas) + balanceCheck.Mul(balanceCheck, st.msg.BlobGasFeeCap) + if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { + return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) + } + // Pay for blobGasUsed * actual blob fee + blobFee := new(big.Int).SetUint64(blobGas) + blobFee.Mul(blobFee, st.evm.Context.BlobBaseFee) + st.state.SubBalance(st.msg.From, blobFee, state.BalanceChangeBurn) + } + return nil +} + func (st *StateTransition) preCheck() error { // Only check transactions that are not fake msg := st.msg @@ -467,8 +507,18 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { st.gasRemaining += refund // Return ETH for remaining gas, exchanged at the original rate. - remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gasRemaining), st.msg.GasPrice) - st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) + gasRemaining := new(big.Int).SetUint64(st.gasRemaining) + if st.msg.GasFeeCap.Sign() == 0 { + remaining := new(big.Int).Mul(gasRemaining, st.msg.GasPrice) + st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) + } else { + // Split into burnt gas and tip gas to refund. + burnt := new(big.Int).Mul(gasRemaining, st.evm.Context.BaseFee) + st.state.AddBalance(st.msg.From, burnt, state.BalanceChangeBurnRefund) + effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) + tip := new(big.Int).Mul(gasRemaining, effectiveTip) + st.state.AddBalance(st.msg.From, tip, state.BalanceChangeGasRefund) + } if st.evm.Config.Tracer != nil && st.gasRemaining > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, vm.GasChangeTxLeftOverReturned) From ddd1b0a63f398a4a5c7733b9468e517d3ea951e6 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 11 Dec 2023 12:55:24 +0330 Subject: [PATCH 33/43] fix gas purchase --- core/state_transition.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index f4a68ec38f..5f88df3273 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -140,6 +140,8 @@ type Message struct { AccessList types.AccessList BlobGasFeeCap *big.Int BlobHashes []common.Hash + // Distinguishes type-2 txes + DynamicFee bool // When SkipAccountChecks is true, the message nonce is not checked against the // account nonce in state. It also disables checking that the sender is an EOA. @@ -162,6 +164,7 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In SkipAccountChecks: false, BlobHashes: tx.BlobHashes(), BlobGasFeeCap: tx.BlobGasFeeCap(), + DynamicFee: tx.Type() == 2 || tx.Type() == 3, } // If baseFee provided, set gasPrice to effectiveGasPrice. if baseFee != nil { @@ -233,7 +236,7 @@ func (st *StateTransition) to() common.Address { } func (st *StateTransition) buyGas() error { - if st.msg.GasFeeCap != nil { + if st.msg.DynamicFee { if err := st.buy1559Gas(); err != nil { return err } @@ -247,6 +250,10 @@ func (st *StateTransition) buyGas() error { return err } } + valueCheck := new(big.Int).Set(st.msg.Value) + if have, want := st.state.GetBalance(st.msg.From), valueCheck; have.Cmp(want) < 0 { + return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) + } if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err @@ -508,16 +515,16 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Return ETH for remaining gas, exchanged at the original rate. gasRemaining := new(big.Int).SetUint64(st.gasRemaining) - if st.msg.GasFeeCap.Sign() == 0 { - remaining := new(big.Int).Mul(gasRemaining, st.msg.GasPrice) - st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) - } else { + if st.msg.DynamicFee { // Split into burnt gas and tip gas to refund. burnt := new(big.Int).Mul(gasRemaining, st.evm.Context.BaseFee) st.state.AddBalance(st.msg.From, burnt, state.BalanceChangeBurnRefund) effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) tip := new(big.Int).Mul(gasRemaining, effectiveTip) st.state.AddBalance(st.msg.From, tip, state.BalanceChangeGasRefund) + } else { + remaining := new(big.Int).Mul(gasRemaining, st.msg.GasPrice) + st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) } if st.evm.Config.Tracer != nil && st.gasRemaining > 0 { From 0484068df5758bae60818567ce1908f4fba800ea Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 11 Dec 2023 14:17:29 +0330 Subject: [PATCH 34/43] fix lint issues --- core/state/metadata.go | 30 +++++++++++++++--------------- eth/tracers/live/printer.go | 1 - eth/tracers/native/prestate.go | 1 - 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/core/state/metadata.go b/core/state/metadata.go index 90ea3d1742..6a2f20a0f9 100644 --- a/core/state/metadata.go +++ b/core/state/metadata.go @@ -22,30 +22,30 @@ type BalanceChangeReason byte const ( BalanceChangeUnspecified BalanceChangeReason = 0 - BalanceChangeRewardMineUncle = 1 - BalanceChangeRewardMineBlock = 2 - BalanceChangeDaoRefundContract = 3 - BalanceChangeDaoAdjustBalance = 4 - BalanceChangeTransfer = 5 - BalanceChangeGenesisBalance = 6 - BalanceChangeGasBuy = 7 - BalanceChangeRewardTransactionFee = 8 - BalanceChangeGasRefund = 9 - BalanceChangeTouchAccount = 10 + BalanceChangeRewardMineUncle BalanceChangeReason = 1 + BalanceChangeRewardMineBlock BalanceChangeReason = 2 + BalanceChangeDaoRefundContract BalanceChangeReason = 3 + BalanceChangeDaoAdjustBalance BalanceChangeReason = 4 + BalanceChangeTransfer BalanceChangeReason = 5 + BalanceChangeGenesisBalance BalanceChangeReason = 6 + BalanceChangeGasBuy BalanceChangeReason = 7 + BalanceChangeRewardTransactionFee BalanceChangeReason = 8 + BalanceChangeGasRefund BalanceChangeReason = 9 + BalanceChangeTouchAccount BalanceChangeReason = 10 // TODO: rename (debit, credit) // BalanceChangeSuicideRefund is added to the recipient as indicated by a selfdestructing account. - BalanceChangeSuicideRefund = 11 + BalanceChangeSuicideRefund BalanceChangeReason = 11 // BalanceChangeSuicideWithdraw is deducted from a contract due to self-destruct. // This can happen either at the point of self-destruction, or at the end of the tx // if ether was sent to contract post-selfdestruct. - BalanceChangeSuicideWithdraw = 12 + BalanceChangeSuicideWithdraw BalanceChangeReason = 12 // BalanceChangeBurn accounts for: // - EIP-1559 burnt fees // - ether that is sent to a self-destructed contract within the same tx (captured at end of tx) // Note it doesn't account for a self-destruct which appoints same contract as recipient. - BalanceChangeBurn = 13 + BalanceChangeBurn BalanceChangeReason = 13 // BalanceChangeBurnRefund is refunded to an account at the end of transaction based on // gas usage from the estimated burn amount. - BalanceChangeBurnRefund = 14 - BalanceChangeWithdrawal = 15 + BalanceChangeBurnRefund BalanceChangeReason = 14 + BalanceChangeWithdrawal BalanceChangeReason = 15 ) diff --git a/eth/tracers/live/printer.go b/eth/tracers/live/printer.go index 422b009e29..727dc3dc16 100644 --- a/eth/tracers/live/printer.go +++ b/eth/tracers/live/printer.go @@ -69,7 +69,6 @@ func (p *Printer) CaptureTxStart(env *vm.EVM, tx *types.Transaction, from common return } fmt.Printf("CaptureTxStart: tx=%s\n", buf) - } func (p *Printer) CaptureTxEnd(receipt *types.Receipt, err error) { diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index b59834ff5d..8608ab69dc 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -61,7 +61,6 @@ type prestateTracer struct { env *vm.EVM pre stateMap post stateMap - create bool to common.Address config prestateTracerConfig interrupt atomic.Bool // Atomic flag to signal execution interruption From 5e82cbec8d2746d5a65b674762e2b4adfa3cd339 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 11 Dec 2023 15:59:23 +0330 Subject: [PATCH 35/43] nil check --- core/state/statedb.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index f951aa490b..7800be8a09 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -485,7 +485,9 @@ func (s *StateDB) SelfDestruct(addr common.Address) { prev: stateObject.selfDestructed, prevbalance: prev, }) - s.logger.OnBalanceChange(addr, prev, n, BalanceChangeSuicideWithdraw) + if s.logger != nil { + s.logger.OnBalanceChange(addr, prev, n, BalanceChangeSuicideWithdraw) + } stateObject.markSelfdestructed() stateObject.data.Balance = n } @@ -876,7 +878,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { obj.deleted = true // If ether was sent to account post-selfdestruct it is burnt. - if bal := obj.Balance(); bal.Sign() != 0 { + if bal := obj.Balance(); bal.Sign() != 0 && s.logger != nil { s.logger.OnBalanceChange(obj.address, bal, new(big.Int), BalanceChangeBurn) } // We need to maintain account deletions explicitly (will remain From 0d254e4983613113a83ca255e8a25bbbd06fc271 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Mon, 11 Dec 2023 16:17:49 +0330 Subject: [PATCH 36/43] fix test --- eth/tracers/internal/tracetest/prestate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 9a9189e399..24dfa17f91 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -118,7 +118,7 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { } statedb.SetLogger(tracer) evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Tracer: tracer}) - msg, err := core.TransactionToMessage(tx, signer, nil) + msg, err := core.TransactionToMessage(tx, signer, test.Genesis.BaseFee) if err != nil { t.Fatalf("failed to prepare transaction for tracing: %v", err) } From a52c80dec53822ddefe668a111892e27b2e35082 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 12 Dec 2023 11:34:41 +0330 Subject: [PATCH 37/43] Fix test case --- .../testdata/prestate_tracer_with_diff_mode/create_failed.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json index e80dad5667..561ead05b6 100644 --- a/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer_with_diff_mode/create_failed.json @@ -83,7 +83,7 @@ }, "post": { "0x808b4da0be6c9512e948521452227efc619bea52": { - "balance": "0x2cd72a36dd031f089", + "balance": "0x2cd987071ba2346b6", "nonce": 1223933 }, "0x8f03f1a3f10c05e7cccf75c1fd10168e06659be7": { From 89d79d6d3c7ce2c1689ada6f68a670ecdc48a0d3 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Tue, 12 Dec 2023 11:48:25 +0330 Subject: [PATCH 38/43] fix t8n tracer nil check --- cmd/evm/internal/t8ntool/execution.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 1ba3c26a8c..ef118ffbcb 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -234,7 +234,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, ) evm := vm.NewEVM(vmContext, txContext, statedb, chainConfig, vmConfig) - tracer.CaptureTxStart(evm, tx, msg.From) + if tracer != nil { + tracer.CaptureTxStart(evm, tx, msg.From) + } // (ret []byte, usedGas uint64, failed bool, err error) msgResult, err := core.ApplyMessage(evm, msg, gaspool) if err != nil { @@ -242,7 +244,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) gaspool.SetGas(prevGas) - tracer.CaptureTxEnd(nil, err) + if tracer != nil { + tracer.CaptureTxEnd(nil, err) + } continue } includedTxs = append(includedTxs, tx) @@ -284,7 +288,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, //receipt.BlockNumber receipt.TransactionIndex = uint(txIndex) receipts = append(receipts, receipt) - tracer.CaptureTxEnd(receipt, nil) + if tracer != nil { + tracer.CaptureTxEnd(receipt, nil) + } } txIndex++ From 54f4bc741d845f23976b0c33311cc01ce081155c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 13 Dec 2023 13:50:04 +0330 Subject: [PATCH 39/43] fix balance check --- core/state_transition.go | 64 ++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 5f88df3273..95a65f1f67 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -236,24 +236,22 @@ func (st *StateTransition) to() common.Address { } func (st *StateTransition) buyGas() error { - if st.msg.DynamicFee { - if err := st.buy1559Gas(); err != nil { - return err - } + if err := st.balanceCheck(); err != nil { + return err + } + if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + st.buyDynamicFee() } else { - if err := st.buyLegacyGas(); err != nil { - return err - } + fee := new(big.Int).SetUint64(st.msg.GasLimit) + fee = fee.Mul(fee, st.msg.GasPrice) + // TODO: rename change reason to indicate tip. + st.state.SubBalance(st.msg.From, fee, state.BalanceChangeGasBuy) } if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { if err := st.buyBlobGas(); err != nil { return err } } - valueCheck := new(big.Int).Set(st.msg.Value) - if have, want := st.state.GetBalance(st.msg.From), valueCheck; have.Cmp(want) < 0 { - return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) - } if err := st.gp.SubGas(st.msg.GasLimit); err != nil { return err @@ -267,50 +265,44 @@ func (st *StateTransition) buyGas() error { return nil } -func (st *StateTransition) buyLegacyGas() error { - mgval := new(big.Int).SetUint64(st.msg.GasLimit) - mgval = mgval.Mul(mgval, st.msg.GasPrice) - balanceCheck := new(big.Int).Set(mgval) +// balanceCheck checks the account can pay all they claimed +// they can pay. +func (st *StateTransition) balanceCheck() error { + balanceCheck := new(big.Int).SetUint64(st.msg.GasLimit) + // GasFeeCap defaults to gas price for legacy txes. + balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) + balanceCheck = balanceCheck.Add(balanceCheck, st.msg.Value) + if st.msg.BlobGasFeeCap != nil { + blobFee := new(big.Int).SetUint64(st.blobGasUsed()) + blobFee = blobFee.Mul(blobFee, st.msg.BlobGasFeeCap) + balanceCheck = balanceCheck.Add(balanceCheck, blobFee) + } if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) } - st.state.SubBalance(st.msg.From, mgval, state.BalanceChangeGasBuy) return nil } -// buy1559Gas purchases gas according to EIP-1559 rules. See: +// buyDynamicFee purchases gas according to EIP-1559 rules. See: // https://eips.ethereum.org/EIPS/eip-1559 -func (st *StateTransition) buy1559Gas() error { - balanceCheck := new(big.Int).SetUint64(st.msg.GasLimit) - balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) - balanceCheck.Add(balanceCheck, st.msg.Value) - if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { - return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) - } - +func (st *StateTransition) buyDynamicFee() { burnt := new(big.Int).SetUint64(st.msg.GasLimit) burnt = burnt.Mul(burnt, st.evm.Context.BaseFee) + // TODO: rename and create new change reason type so it's possible to calculate total fee paid. st.state.SubBalance(st.msg.From, burnt, state.BalanceChangeBurn) effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) tipVal := new(big.Int).SetUint64(st.msg.GasLimit) tipVal = tipVal.Mul(tipVal, effectiveTip) + // TODO: rename to indicate tip. st.state.SubBalance(st.msg.From, tipVal, state.BalanceChangeGasBuy) - - return nil } // buyBlobGas purchases blob gas as per: // https://eips.ethereum.org/EIPS/eip-4844 func (st *StateTransition) buyBlobGas() error { if blobGas := st.blobGasUsed(); blobGas > 0 { - // Check that the user has enough funds to cover blobGasUsed * tx.BlobGasFeeCap - balanceCheck := new(big.Int).SetUint64(blobGas) - balanceCheck.Mul(balanceCheck, st.msg.BlobGasFeeCap) - if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { - return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) - } - // Pay for blobGasUsed * actual blob fee + // Pay for blobGasUsed * effective blob fee blobFee := new(big.Int).SetUint64(blobGas) blobFee.Mul(blobFee, st.evm.Context.BlobBaseFee) st.state.SubBalance(st.msg.From, blobFee, state.BalanceChangeBurn) @@ -515,15 +507,17 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Return ETH for remaining gas, exchanged at the original rate. gasRemaining := new(big.Int).SetUint64(st.gasRemaining) - if st.msg.DynamicFee { + if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { // Split into burnt gas and tip gas to refund. burnt := new(big.Int).Mul(gasRemaining, st.evm.Context.BaseFee) + // TODO: rename to unburnt st.state.AddBalance(st.msg.From, burnt, state.BalanceChangeBurnRefund) effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) tip := new(big.Int).Mul(gasRemaining, effectiveTip) st.state.AddBalance(st.msg.From, tip, state.BalanceChangeGasRefund) } else { remaining := new(big.Int).Mul(gasRemaining, st.msg.GasPrice) + // TODO: rename to unusedTip st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) } From 64e35e93dd8a4e4bb585f46df7a8cad37e01a9c3 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 13 Dec 2023 16:44:56 +0330 Subject: [PATCH 40/43] revert buyGas refactor --- core/state_transition.go | 90 ++++++++++++---------------------------- 1 file changed, 27 insertions(+), 63 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 95a65f1f67..311bd7fda1 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -236,77 +236,41 @@ func (st *StateTransition) to() common.Address { } func (st *StateTransition) buyGas() error { - if err := st.balanceCheck(); err != nil { - return err - } - if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { - st.buyDynamicFee() - } else { - fee := new(big.Int).SetUint64(st.msg.GasLimit) - fee = fee.Mul(fee, st.msg.GasPrice) - // TODO: rename change reason to indicate tip. - st.state.SubBalance(st.msg.From, fee, state.BalanceChangeGasBuy) + mgval := new(big.Int).SetUint64(st.msg.GasLimit) + mgval = mgval.Mul(mgval, st.msg.GasPrice) + balanceCheck := new(big.Int).Set(mgval) + if st.msg.GasFeeCap != nil { + balanceCheck.SetUint64(st.msg.GasLimit) + balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) + balanceCheck.Add(balanceCheck, st.msg.Value) } if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { - if err := st.buyBlobGas(); err != nil { - return err + if blobGas := st.blobGasUsed(); blobGas > 0 { + // Check that the user has enough funds to cover blobGasUsed * tx.BlobGasFeeCap + blobBalanceCheck := new(big.Int).SetUint64(blobGas) + blobBalanceCheck.Mul(blobBalanceCheck, st.msg.BlobGasFeeCap) + balanceCheck.Add(balanceCheck, blobBalanceCheck) + // Pay for blobGasUsed * actual blob fee + blobFee := new(big.Int).SetUint64(blobGas) + blobFee.Mul(blobFee, st.evm.Context.BlobBaseFee) + mgval.Add(mgval, blobFee) } } - - if err := st.gp.SubGas(st.msg.GasLimit); err != nil { - return err - } - if st.evm.Config.Tracer != nil { - st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, vm.GasChangeTxInitialBalance) - } - st.gasRemaining += st.msg.GasLimit - st.initialGas = st.msg.GasLimit - - return nil -} - -// balanceCheck checks the account can pay all they claimed -// they can pay. -func (st *StateTransition) balanceCheck() error { - balanceCheck := new(big.Int).SetUint64(st.msg.GasLimit) - // GasFeeCap defaults to gas price for legacy txes. - balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) - balanceCheck = balanceCheck.Add(balanceCheck, st.msg.Value) - if st.msg.BlobGasFeeCap != nil { - blobFee := new(big.Int).SetUint64(st.blobGasUsed()) - blobFee = blobFee.Mul(blobFee, st.msg.BlobGasFeeCap) - balanceCheck = balanceCheck.Add(balanceCheck, blobFee) - } if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) } - return nil -} - -// buyDynamicFee purchases gas according to EIP-1559 rules. See: -// https://eips.ethereum.org/EIPS/eip-1559 -func (st *StateTransition) buyDynamicFee() { - burnt := new(big.Int).SetUint64(st.msg.GasLimit) - burnt = burnt.Mul(burnt, st.evm.Context.BaseFee) - // TODO: rename and create new change reason type so it's possible to calculate total fee paid. - st.state.SubBalance(st.msg.From, burnt, state.BalanceChangeBurn) - - effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) - tipVal := new(big.Int).SetUint64(st.msg.GasLimit) - tipVal = tipVal.Mul(tipVal, effectiveTip) - // TODO: rename to indicate tip. - st.state.SubBalance(st.msg.From, tipVal, state.BalanceChangeGasBuy) -} - -// buyBlobGas purchases blob gas as per: -// https://eips.ethereum.org/EIPS/eip-4844 -func (st *StateTransition) buyBlobGas() error { - if blobGas := st.blobGasUsed(); blobGas > 0 { - // Pay for blobGasUsed * effective blob fee - blobFee := new(big.Int).SetUint64(blobGas) - blobFee.Mul(blobFee, st.evm.Context.BlobBaseFee) - st.state.SubBalance(st.msg.From, blobFee, state.BalanceChangeBurn) + if err := st.gp.SubGas(st.msg.GasLimit); err != nil { + return err } + + if st.evm.Config.Tracer != nil { + st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, vm.GasChangeTxInitialBalance) + } + + st.gasRemaining += st.msg.GasLimit + + st.initialGas = st.msg.GasLimit + st.state.SubBalance(st.msg.From, mgval, state.BalanceChangeGasBuy) return nil } From 8b3180d8a72f9c04d24df0bcfd338fade12dfb22 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 13 Dec 2023 17:51:19 +0330 Subject: [PATCH 41/43] revert gas refund --- core/state_transition.go | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/core/state_transition.go b/core/state_transition.go index 311bd7fda1..165fbd853e 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -470,20 +470,8 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { st.gasRemaining += refund // Return ETH for remaining gas, exchanged at the original rate. - gasRemaining := new(big.Int).SetUint64(st.gasRemaining) - if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { - // Split into burnt gas and tip gas to refund. - burnt := new(big.Int).Mul(gasRemaining, st.evm.Context.BaseFee) - // TODO: rename to unburnt - st.state.AddBalance(st.msg.From, burnt, state.BalanceChangeBurnRefund) - effectiveTip := cmath.BigMin(st.msg.GasTipCap, new(big.Int).Sub(st.msg.GasFeeCap, st.evm.Context.BaseFee)) - tip := new(big.Int).Mul(gasRemaining, effectiveTip) - st.state.AddBalance(st.msg.From, tip, state.BalanceChangeGasRefund) - } else { - remaining := new(big.Int).Mul(gasRemaining, st.msg.GasPrice) - // TODO: rename to unusedTip - st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) - } + remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gasRemaining), st.msg.GasPrice) + st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) if st.evm.Config.Tracer != nil && st.gasRemaining > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, vm.GasChangeTxLeftOverReturned) From 2d14bb573c90a427b3454e446464655cc3f8092f Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Wed, 13 Dec 2023 17:52:56 +0330 Subject: [PATCH 42/43] rename and document balance change reasons --- cmd/evm/internal/t8ntool/execution.go | 6 +-- consensus/ethash/consensus.go | 4 +- consensus/misc/dao.go | 4 +- core/genesis.go | 4 +- core/state/metadata.go | 67 +++++++++++++++++---------- core/state/statedb.go | 4 +- core/state_transition.go | 6 +-- core/vm/instructions.go | 6 +-- 8 files changed, 59 insertions(+), 42 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index ef118ffbcb..df46e298d8 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -316,9 +316,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) reward.Div(reward, big.NewInt(8)) - statedb.AddBalance(ommer.Address, reward, state.BalanceChangeRewardMineUncle) + statedb.AddBalance(ommer.Address, reward, state.BalanceIncreaseRewardMineUncle) } - statedb.AddBalance(pre.Env.Coinbase, minerReward, state.BalanceChangeRewardMineBlock) + statedb.AddBalance(pre.Env.Coinbase, minerReward, state.BalanceIncreaseRewardMineBlock) } // Apply withdrawals for _, w := range pre.Env.Withdrawals { @@ -367,7 +367,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB for addr, a := range accounts { statedb.SetCode(addr, a.Code) statedb.SetNonce(addr, a.Nonce) - statedb.SetBalance(addr, a.Balance, state.BalanceChangeGenesisBalance) + statedb.SetBalance(addr, a.Balance, state.BalanceIncreaseGenesisBalance) for k, v := range a.Storage { statedb.SetState(addr, k, v) } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 4b919ee591..17346384e1 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -586,10 +586,10 @@ func accumulateRewards(config *params.ChainConfig, stateDB *state.StateDB, heade r.Sub(r, header.Number) r.Mul(r, blockReward) r.Div(r, big8) - stateDB.AddBalance(uncle.Coinbase, r, state.BalanceChangeRewardMineUncle) + stateDB.AddBalance(uncle.Coinbase, r, state.BalanceIncreaseRewardMineUncle) r.Div(blockReward, big32) reward.Add(reward, r) } - stateDB.AddBalance(header.Coinbase, reward, state.BalanceChangeRewardMineBlock) + stateDB.AddBalance(header.Coinbase, reward, state.BalanceIncreaseRewardMineBlock) } diff --git a/consensus/misc/dao.go b/consensus/misc/dao.go index 7f0895eb26..bc393a9cbc 100644 --- a/consensus/misc/dao.go +++ b/consensus/misc/dao.go @@ -80,7 +80,7 @@ func ApplyDAOHardFork(statedb *state.StateDB) { // Move every DAO account and extra-balance account funds into the refund contract for _, addr := range params.DAODrainList() { - statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr), state.BalanceChangeDaoRefundContract) - statedb.SetBalance(addr, new(big.Int), state.BalanceChangeDaoAdjustBalance) + statedb.AddBalance(params.DAORefundContract, statedb.GetBalance(addr), state.BalanceIncreaseDaoContract) + statedb.SetBalance(addr, new(big.Int), state.BalanceDecreaseDaoAccount) } } diff --git a/core/genesis.go b/core/genesis.go index ed2f1eb503..59f06007b2 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -142,7 +142,7 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { } for addr, account := range *ga { if account.Balance != nil { - statedb.AddBalance(addr, account.Balance, state.BalanceChangeGenesisBalance) + statedb.AddBalance(addr, account.Balance, state.BalanceIncreaseGenesisBalance) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) @@ -165,7 +165,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas if account.Balance != nil { // This is not actually logged via tracer because OnGenesisBlock // already captures the allocations. - statedb.AddBalance(addr, account.Balance, state.BalanceChangeGenesisBalance) + statedb.AddBalance(addr, account.Balance, state.BalanceIncreaseGenesisBalance) } statedb.SetCode(addr, account.Code) statedb.SetNonce(addr, account.Nonce) diff --git a/core/state/metadata.go b/core/state/metadata.go index 6a2f20a0f9..6e7a717794 100644 --- a/core/state/metadata.go +++ b/core/state/metadata.go @@ -21,31 +21,48 @@ package state type BalanceChangeReason byte const ( - BalanceChangeUnspecified BalanceChangeReason = 0 - BalanceChangeRewardMineUncle BalanceChangeReason = 1 - BalanceChangeRewardMineBlock BalanceChangeReason = 2 - BalanceChangeDaoRefundContract BalanceChangeReason = 3 - BalanceChangeDaoAdjustBalance BalanceChangeReason = 4 - BalanceChangeTransfer BalanceChangeReason = 5 - BalanceChangeGenesisBalance BalanceChangeReason = 6 - BalanceChangeGasBuy BalanceChangeReason = 7 - BalanceChangeRewardTransactionFee BalanceChangeReason = 8 - BalanceChangeGasRefund BalanceChangeReason = 9 - BalanceChangeTouchAccount BalanceChangeReason = 10 - // TODO: rename (debit, credit) - // BalanceChangeSuicideRefund is added to the recipient as indicated by a selfdestructing account. - BalanceChangeSuicideRefund BalanceChangeReason = 11 - // BalanceChangeSuicideWithdraw is deducted from a contract due to self-destruct. + BalanceChangeUnspecified BalanceChangeReason = 0 + + // Issuance + // BalanceIncreaseRewardMineUncle is a reward for mining an uncle block. + BalanceIncreaseRewardMineUncle BalanceChangeReason = 1 + // BalanceIncreaseRewardMineBlock is a reward for mining a block. + BalanceIncreaseRewardMineBlock BalanceChangeReason = 2 + // BalanceIncreaseWithdrawal is ether withdrawn from the beacon chain. + BalanceChangeWithdrawal BalanceChangeReason = 3 + // BalanceIncreaseGenesisBalance is ether allocated at the genesis block. + BalanceIncreaseGenesisBalance BalanceChangeReason = 4 + + // Transaction fees + // BalanceIncreaseRewardTransactionFee is the transaction tip increasing block builder's balance. + BalanceIncreaseRewardTransactionFee BalanceChangeReason = 5 + // BalanceDecreaseGasBuy is spent to purchase gas for execution a transaction. + // Part of this gas will be burnt as per EIP-1559 rules. + BalanceDecreaseGasBuy BalanceChangeReason = 6 + // BalanceIncreaseGasReturn is ether returned for unused gas at the end of execution. + BalanceIncreaseGasReturn BalanceChangeReason = 7 + + // DAO fork + // BalanceIncreaseDaoContract is ether sent to the DAO refund contract. + BalanceIncreaseDaoContract BalanceChangeReason = 8 + // BalanceDecreaseDaoAccount is ether taken from a DAO account to be moved to the refund contract. + BalanceDecreaseDaoAccount BalanceChangeReason = 9 + + // BalanceChangeTransfer is ether transfered via a call. + // it is a decrease for the sender and an increase for the recipient. + BalanceChangeTransfer BalanceChangeReason = 10 + // BalanceChangeTouchAccount is a transfer of zero value. It is only there to + // touch-create an account. + BalanceChangeTouchAccount BalanceChangeReason = 11 + + // BalanceIncreaseSelfdestruct is added to the recipient as indicated by a selfdestructing account. + BalanceIncreaseSelfdestruct BalanceChangeReason = 12 + // BalanceDecreaseSelfdestruct is deducted from a contract due to self-destruct. // This can happen either at the point of self-destruction, or at the end of the tx // if ether was sent to contract post-selfdestruct. - BalanceChangeSuicideWithdraw BalanceChangeReason = 12 - // BalanceChangeBurn accounts for: - // - EIP-1559 burnt fees - // - ether that is sent to a self-destructed contract within the same tx (captured at end of tx) - // Note it doesn't account for a self-destruct which appoints same contract as recipient. - BalanceChangeBurn BalanceChangeReason = 13 - // BalanceChangeBurnRefund is refunded to an account at the end of transaction based on - // gas usage from the estimated burn amount. - BalanceChangeBurnRefund BalanceChangeReason = 14 - BalanceChangeWithdrawal BalanceChangeReason = 15 + BalanceDecreaseSelfdestruct BalanceChangeReason = 13 + // BalanceDecreaseSelfdestructBurn is ether that is sent to an already self-destructed + // account within the same tx (captured at end of tx). + // Note it doesn't account for a self-destruct which appoints itself as recipient. + BalanceDecreaseSelfdestructBurn BalanceChangeReason = 14 ) diff --git a/core/state/statedb.go b/core/state/statedb.go index 7800be8a09..ff33096f9c 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -486,7 +486,7 @@ func (s *StateDB) SelfDestruct(addr common.Address) { prevbalance: prev, }) if s.logger != nil { - s.logger.OnBalanceChange(addr, prev, n, BalanceChangeSuicideWithdraw) + s.logger.OnBalanceChange(addr, prev, n, BalanceDecreaseSelfdestruct) } stateObject.markSelfdestructed() stateObject.data.Balance = n @@ -879,7 +879,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // If ether was sent to account post-selfdestruct it is burnt. if bal := obj.Balance(); bal.Sign() != 0 && s.logger != nil { - s.logger.OnBalanceChange(obj.address, bal, new(big.Int), BalanceChangeBurn) + s.logger.OnBalanceChange(obj.address, bal, new(big.Int), BalanceDecreaseSelfdestructBurn) } // We need to maintain account deletions explicitly (will remain // set indefinitely). Note only the first occurred self-destruct diff --git a/core/state_transition.go b/core/state_transition.go index 165fbd853e..880e7f81fb 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -270,7 +270,7 @@ func (st *StateTransition) buyGas() error { st.gasRemaining += st.msg.GasLimit st.initialGas = st.msg.GasLimit - st.state.SubBalance(st.msg.From, mgval, state.BalanceChangeGasBuy) + st.state.SubBalance(st.msg.From, mgval, state.BalanceDecreaseGasBuy) return nil } @@ -445,7 +445,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { } else { fee := new(big.Int).SetUint64(st.gasUsed()) fee.Mul(fee, effectiveTip) - st.state.AddBalance(st.evm.Context.Coinbase, fee, state.BalanceChangeRewardTransactionFee) + st.state.AddBalance(st.evm.Context.Coinbase, fee, state.BalanceIncreaseRewardTransactionFee) } return &ExecutionResult{ @@ -471,7 +471,7 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { // Return ETH for remaining gas, exchanged at the original rate. remaining := new(big.Int).Mul(new(big.Int).SetUint64(st.gasRemaining), st.msg.GasPrice) - st.state.AddBalance(st.msg.From, remaining, state.BalanceChangeGasRefund) + st.state.AddBalance(st.msg.From, remaining, state.BalanceIncreaseGasReturn) if st.evm.Config.Tracer != nil && st.gasRemaining > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, vm.GasChangeTxLeftOverReturned) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 13f7425f67..a70b4bea77 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -855,7 +855,7 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext } beneficiary := scope.Stack.pop() balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) - interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceChangeSuicideRefund) + interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceIncreaseSelfdestruct) interpreter.evm.StateDB.SelfDestruct(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) @@ -870,8 +870,8 @@ func opSelfdestruct6780(pc *uint64, interpreter *EVMInterpreter, scope *ScopeCon } beneficiary := scope.Stack.pop() balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) - interpreter.evm.StateDB.SubBalance(scope.Contract.Address(), balance, state.BalanceChangeSuicideWithdraw) - interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceChangeSuicideRefund) + interpreter.evm.StateDB.SubBalance(scope.Contract.Address(), balance, state.BalanceDecreaseSelfdestruct) + interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance, state.BalanceIncreaseSelfdestruct) interpreter.evm.StateDB.Selfdestruct6780(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) From 68bd6c6b317c0dcfa479aa3fd9328ed8583db57a Mon Sep 17 00:00:00 2001 From: Matthieu Vachon Date: Thu, 11 Jan 2024 06:24:52 -0500 Subject: [PATCH 43/43] Removed the `OnGenesisBlock` in `genesis.Commit` (#20) This is not actually required as the `blockchain` does emit the `OnGenesisBlock` itself if starting a sync from scratch. --- cmd/geth/chaincmd.go | 2 +- core/blockchain.go | 2 +- core/chain_makers.go | 2 +- core/genesis.go | 19 ++++++++----------- core/genesis_test.go | 10 +++++----- core/headerchain_test.go | 2 +- eth/filters/filter_test.go | 2 +- tests/block_test_util.go | 2 +- 8 files changed, 19 insertions(+), 22 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index e170982142..a7715e6e7a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -202,7 +202,7 @@ func initGenesis(ctx *cli.Context) error { triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle()) defer triedb.Close() - _, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides, nil) + _, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides) if err != nil { utils.Fatalf("Failed to write genesis block: %v", err) } diff --git a/core/blockchain.go b/core/blockchain.go index e7ba251c8f..1484b8650a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -295,7 +295,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis // Setup the genesis block, commit the provided genesis specification // to database if the genesis block is not present yet, or load the // stored one from database. - chainConfig, genesisHash, genesisErr := SetupGenesisBlockWithOverride(db, triedb, genesis, overrides, logger) + chainConfig, genesisHash, genesisErr := SetupGenesisBlockWithOverride(db, triedb, genesis, overrides) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/core/chain_makers.go b/core/chain_makers.go index f0d21e9c45..2949162bfe 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -409,7 +409,7 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, triedb := trie.NewDatabase(db, trie.HashDefaults) defer triedb.Close() - _, err := genesis.Commit(db, triedb, nil) + _, err := genesis.Commit(db, triedb) if err != nil { panic(err) } diff --git a/core/genesis.go b/core/genesis.go index 59f06007b2..b28d403b22 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -156,7 +156,7 @@ func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { // flush is very similar with hash, but the main difference is all the generated // states will be persisted into the given database. Also, the genesis state // specification will be flushed as well. -func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, bcLogger BlockchainLogger) error { +func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { return err @@ -307,10 +307,10 @@ type ChainOverrides struct { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, triedb *trie.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, triedb, genesis, nil, nil) + return SetupGenesisBlockWithOverride(db, triedb, genesis, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, genesis *Genesis, overrides *ChainOverrides, bcLogger BlockchainLogger) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -335,7 +335,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen } applyOverrides(genesis.Config) - block, err := genesis.Commit(db, triedb, bcLogger) + block, err := genesis.Commit(db, triedb) if err != nil { return genesis.Config, common.Hash{}, err } @@ -356,7 +356,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } - block, err := genesis.Commit(db, triedb, bcLogger) + block, err := genesis.Commit(db, triedb) if err != nil { return genesis.Config, hash, err } @@ -524,7 +524,7 @@ func (g *Genesis) ToBlock() *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. -func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database, bcLogger BlockchainLogger) (*types.Block, error) { +func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block, error) { block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") @@ -539,13 +539,10 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database, bcLogger Bloc if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - if bcLogger != nil { - bcLogger.OnGenesisBlock(block, g.Alloc) - } // All the checks has passed, flush the states derived from the genesis // specification as well as the specification itself into the provided // database. - if err := g.Alloc.flush(db, triedb, block.Hash(), bcLogger); err != nil { + if err := g.Alloc.flush(db, triedb, block.Hash()); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -564,7 +561,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database, bcLogger Bloc // Note the state changes will be committed in hash-based scheme, use Commit // if path-scheme is preferred. func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block { - block, err := g.Commit(db, triedb, nil) + block, err := g.Commit(db, triedb) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index 8d97430123..1d85b510ca 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -38,7 +38,7 @@ func TestInvalidCliqueConfig(t *testing.T) { block := DefaultGoerliGenesisBlock() block.ExtraData = []byte{} db := rawdb.NewMemoryDatabase() - if _, err := block.Commit(db, trie.NewDatabase(db, nil), nil); err == nil { + if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil { t.Fatal("Expected error on invalid clique config") } } @@ -97,7 +97,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == nil", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { tdb := trie.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb, nil) + customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, nil) }, wantHash: customghash, @@ -107,7 +107,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "custom block in DB, genesis == goerli", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { tdb := trie.NewDatabase(db, newDbConfig(scheme)) - customg.Commit(db, tdb, nil) + customg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock()) }, wantErr: &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash}, @@ -118,7 +118,7 @@ func testSetupGenesis(t *testing.T, scheme string) { name: "compatible config in DB", fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) { tdb := trie.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb, nil) + oldcustomg.Commit(db, tdb) return SetupGenesisBlock(db, tdb, &customg) }, wantHash: customghash, @@ -130,7 +130,7 @@ func testSetupGenesis(t *testing.T, scheme string) { // Commit the 'old' genesis block with Homestead transition at #2. // Advance to block #4, past the homestead transition block of customg. tdb := trie.NewDatabase(db, newDbConfig(scheme)) - oldcustomg.Commit(db, tdb, nil) + oldcustomg.Commit(db, tdb) bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil) defer bc.Stop() diff --git a/core/headerchain_test.go b/core/headerchain_test.go index 9e901671e8..2c0323e6f7 100644 --- a/core/headerchain_test.go +++ b/core/headerchain_test.go @@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) { db = rawdb.NewMemoryDatabase() gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges} ) - gspec.Commit(db, trie.NewDatabase(db, nil), nil) + gspec.Commit(db, trie.NewDatabase(db, nil)) hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false }) if err != nil { t.Fatal(err) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index a14bc3b20f..1db917c960 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -180,7 +180,7 @@ func TestFilters(t *testing.T) { // Hack: GenerateChainWithGenesis creates a new db. // Commit the genesis manually and use GenerateChain. - _, err = gspec.Commit(db, trie.NewDatabase(db, nil), nil) + _, err = gspec.Commit(db, trie.NewDatabase(db, nil)) if err != nil { t.Fatal(err) } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 765935867c..e0130be48a 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -128,7 +128,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po // Commit genesis state gspec := t.genesis(config) triedb := trie.NewDatabase(db, tconf) - gblock, err := gspec.Commit(db, triedb, nil) + gblock, err := gspec.Commit(db, triedb) if err != nil { return err }