Merge branch 'ethereum:master' into build/go1.24.2

This commit is contained in:
levisyin 2025-04-04 00:51:07 +08:00 committed by GitHub
commit e9032e1f00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
61 changed files with 1284 additions and 853 deletions

View file

@ -1,31 +1,25 @@
# This file configures github.com/golangci/golangci-lint.
version: '2'
run:
timeout: 20m
tests: true
linters:
disable-all: true
default: none
enable:
- goimports
- gosimple
- govet
- ineffassign
- misspell
- unconvert
- typecheck
- unused
- staticcheck
- bidichk
- durationcheck
- copyloopvar
- whitespace
- revive # only certain checks enabled
- durationcheck
- gocheckcompilerdirectives
- reassign
- govet
- ineffassign
- mirror
- misspell
- reassign
- revive # only certain checks enabled
- staticcheck
- unconvert
- unused
- usetesting
- whitespace
### linters we tried and will not be using:
###
# - structcheck # lots of false positives
@ -36,44 +30,67 @@ linters:
# - exhaustive # silly check
# - makezero # false positives
# - nilerr # several intentional
linters-settings:
gofmt:
simplify: true
revive:
enable-all-rules: false
# here we enable specific useful rules
# see https://golangci-lint.run/usage/linters/#revive for supported rules
settings:
staticcheck:
checks:
# disable Quickfixes
- -QF1*
revive:
enable-all-rules: false
# here we enable specific useful rules
# see https://golangci-lint.run/usage/linters/#revive for supported rules
rules:
- name: receiver-naming
severity: warning
disabled: false
exclude:
- ''
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
rules:
- name: receiver-naming
severity: warning
disabled: false
exclude: [""]
issues:
# default is true. Enables skipping of directories:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
exclude-dirs-use-default: true
exclude-files:
- core/genesis_alloc.go
exclude-rules:
- path: crypto/bn256/cloudflare/optate.go
linters:
- deadcode
- staticcheck
- path: crypto/bn256/
linters:
- revive
- path: cmd/utils/flags.go
text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: cmd/utils/flags.go
text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: internal/build/pgp.go
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
- path: core/vm/contracts.go
text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
exclude:
- 'SA1019: event.TypeMux is deprecated: use Feed'
- 'SA1019: strings.Title is deprecated'
- 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
- 'SA1029: should not use built-in type string as key for value'
- linters:
- deadcode
- staticcheck
path: crypto/bn256/cloudflare/optate.go
- linters:
- revive
path: crypto/bn256/
- path: cmd/utils/flags.go
text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: cmd/utils/flags.go
text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: internal/build/pgp.go
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
- path: core/vm/contracts.go
text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
- path: (.+)\.go$
text: 'SA1019: event.TypeMux is deprecated: use Feed'
- path: (.+)\.go$
text: 'SA1019: strings.Title is deprecated'
- path: (.+)\.go$
text: 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
- path: (.+)\.go$
text: 'SA1029: should not use built-in type string as key for value'
paths:
- core/genesis_alloc.go
- third_party$
- builtin$
- examples$
formatters:
enable:
- goimports
settings:
gofmt:
simplify: true
exclusions:
generated: lax
paths:
- core/genesis_alloc.go
- third_party$
- builtin$
- examples$

View file

@ -23,9 +23,11 @@ import (
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
)
@ -46,7 +48,13 @@ func NewClient(config params.ClientConfig) *Client {
var (
db = memorydb.New()
committeeChain = light.NewCommitteeChain(db, &config.ChainConfig, config.Threshold, !config.NoFilter)
headTracker = light.NewHeadTracker(committeeChain, config.Threshold)
headTracker = light.NewHeadTracker(committeeChain, config.Threshold, func(checkpoint common.Hash) {
if saved, err := config.SaveCheckpointToFile(checkpoint); saved {
log.Debug("Saved beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint)
} else if err != nil {
log.Error("Failed to save beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint, "error", err)
}
})
)
headSync := sync.NewHeadSync(headTracker, committeeChain)

View file

@ -21,7 +21,9 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
@ -38,13 +40,15 @@ type HeadTracker struct {
hasFinalityUpdate bool
prefetchHead types.HeadInfo
changeCounter uint64
saveCheckpoint func(common.Hash)
}
// NewHeadTracker creates a new HeadTracker.
func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker {
func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int, saveCheckpoint func(common.Hash)) *HeadTracker {
return &HeadTracker{
committeeChain: committeeChain,
minSignerCount: minSignerCount,
saveCheckpoint: saveCheckpoint,
}
}
@ -100,6 +104,9 @@ func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error
if replace {
h.finalityUpdate, h.hasFinalityUpdate = update, true
h.changeCounter++
if h.saveCheckpoint != nil && update.Finalized.Slot%params.EpochLength == 0 {
h.saveCheckpoint(update.Finalized.Hash())
}
}
return replace, err
}

View file

@ -54,6 +54,7 @@ type ChainConfig struct {
GenesisValidatorsRoot common.Hash // Root hash of the genesis validator set, used for signature domain calculation
Forks Forks
Checkpoint common.Hash
CheckpointFile string
}
// ForkAtEpoch returns the latest active fork at the given epoch.
@ -211,3 +212,36 @@ func (f Forks) Less(i, j int) bool {
}
return f[i].knownIndex < f[j].knownIndex
}
// SetCheckpointFile sets the checkpoint import/export file name and attempts to
// read the checkpoint from the file if it already exists. It returns true if
// a checkpoint has been loaded.
func (c *ChainConfig) SetCheckpointFile(checkpointFile string) (bool, error) {
c.CheckpointFile = checkpointFile
file, err := os.ReadFile(checkpointFile)
if os.IsNotExist(err) {
return false, nil // did not load checkpoint
}
if err != nil {
return false, fmt.Errorf("failed to read beacon checkpoint file: %v", err)
}
cp, err := hexutil.Decode(string(file))
if err != nil {
return false, fmt.Errorf("failed to decode hex string in beacon checkpoint file: %v", err)
}
if len(cp) != 32 {
return false, fmt.Errorf("invalid hex string length in beacon checkpoint file: %d", len(cp))
}
copy(c.Checkpoint[:len(cp)], cp)
return true, nil
}
// SaveCheckpointToFile saves the given checkpoint to file if a checkpoint
// import/export file has been specified.
func (c *ChainConfig) SaveCheckpointToFile(checkpoint common.Hash) (bool, error) {
if c.CheckpointFile == "" {
return false, nil
}
err := os.WriteFile(c.CheckpointFile, []byte(checkpoint.Hex()), 0600)
return err == nil, err
}

View file

@ -54,60 +54,60 @@ acefb191e72fea0bdb1a3f5f8f6f5ab18b42b3bbce0c7183f189f25953aff275 go1.24.2.windo
ab267f7f9a3366d48d7664be9e627ce3e63273231430cce5f7783fb910f14148 go1.24.2.windows-arm64.zip
d187bfe539356c39573d2f46766d1d08122b4f33da00fd14d12485fa9e241ff5 go1.24.2.windows-arm64.msi
# version:golangci 1.64.6
# version:golangci 2.0.2
# https://github.com/golangci/golangci-lint/releases/
# https://github.com/golangci/golangci-lint/releases/download/v1.64.6/
08f9459e7125fed2612abd71596e04d172695921aff82120d1c1e5e9b6fff2a3 golangci-lint-1.64.6-darwin-amd64.tar.gz
8c10d0c7c3935b8c2269d628b6a06a8f48d8fb4cc31af02fe4ce0cd18b0704c1 golangci-lint-1.64.6-darwin-arm64.tar.gz
c07fcabb55deb8d2c4d390025568e76162d7f91b1d14bd2311be45d8d440a624 golangci-lint-1.64.6-freebsd-386.tar.gz
8ed1ef1102e1a42983ffcaae8e06de6a540334c3a69e205c610b8a7c92d469cd golangci-lint-1.64.6-freebsd-amd64.tar.gz
8f8dda66d1b4a85cc8a1daf1b69af6559c3eb0a41dd8033148a9ad85cfc0e1d9 golangci-lint-1.64.6-freebsd-armv6.tar.gz
59e8ca1fa254661b2a55e96515b14a10cd02221d443054deac5b28c3c3e12d6b golangci-lint-1.64.6-freebsd-armv7.tar.gz
e3d323d5f132e9b7593141bfe1d19c8460e65a55cea1008ec96945fed563f981 golangci-lint-1.64.6-illumos-amd64.tar.gz
5ce910f2a864c5fbeb32a30cbd506e1b2ef215f7a0f422cd5be6370b13db6f03 golangci-lint-1.64.6-linux-386.deb
2caab682a26b9a5fb94ba24e3a7e1404fa9eab2c12e36ae1c5548d80a1be190c golangci-lint-1.64.6-linux-386.rpm
2d82d0a4716e6d9b70c95103054855cb4b5f20f7bbdee42297f0189955bd14b6 golangci-lint-1.64.6-linux-386.tar.gz
9cd82503e9841abcaa57663fc899587fe90363c26d86a793a98d3080fd25e907 golangci-lint-1.64.6-linux-amd64.deb
981aaca5e5202d4fbb162ec7080490eb67ef5d32add5fb62fb02210597acc9da golangci-lint-1.64.6-linux-amd64.rpm
71e290acbacb7b3ba4f68f0540fb74dc180c4336ac8a6f3bdcd7fcc48b15148d golangci-lint-1.64.6-linux-amd64.tar.gz
718016bb06c1f598a8d23c7964e2643de621dbe5808688cb38857eb0bb773c84 golangci-lint-1.64.6-linux-arm64.deb
ddc0e7b4b10b47cf894aea157e89e3674bbb60f8f5c480110c21c49cc2c1634d golangci-lint-1.64.6-linux-arm64.rpm
99a7ff13dec7a8781a68408b6ecb8a1c5e82786cba3189eaa91d5cdcc24362ce golangci-lint-1.64.6-linux-arm64.tar.gz
90e360f89c244394912b8709fb83a900b6d56cf19141df2afaf9e902ef3057b1 golangci-lint-1.64.6-linux-armv6.deb
46546ff7c98d873ffcdbee06b39dc1024fc08db4fbf1f6309360a44cf976b5c2 golangci-lint-1.64.6-linux-armv6.rpm
e45c1a42868aca0b0ee54d14fb89da216f3b4409367cd7bce3b5f36788b4fc92 golangci-lint-1.64.6-linux-armv6.tar.gz
3cf6ddbbbf358db3de4b64a24f9683bbe2da3f003cfdee86cb610124b57abafb golangci-lint-1.64.6-linux-armv7.deb
508b6712710da10f11aab9ea5e63af415c932dfe7fff718e649d3100b838f797 golangci-lint-1.64.6-linux-armv7.rpm
da9a8bbee86b4dfee73fbc17e0856ec84c5b04919eb09bf3dd5904c39ce41753 golangci-lint-1.64.6-linux-armv7.tar.gz
ad496a58284e1e9c8ac6f993fec429dcd96c85a8c4715dbb6530a5af0dae7fbd golangci-lint-1.64.6-linux-loong64.deb
3bd70fa737b224750254dce616d9a188570e49b894b0cdb2fd04155e2c061350 golangci-lint-1.64.6-linux-loong64.rpm
a535af973499729f2215a84303eb0de61399057aad6901ddea1b4f73f68d5f2c golangci-lint-1.64.6-linux-loong64.tar.gz
6ad2a1cd37ca30303a488abfca2de52aff57519901c6d8d2608656fe9fb05292 golangci-lint-1.64.6-linux-mips64.deb
5f18369f0ca010a02c267352ebe6e3e0514c6debff49899c9e5520906c0da287 golangci-lint-1.64.6-linux-mips64.rpm
3449d6c13307b91b0e8817f8911c07c3412cdb6931b8d101e42db3e9762e91ad golangci-lint-1.64.6-linux-mips64.tar.gz
d4712a348f65dcaf9f6c58f1cfd6d0984d54a902873dca5e76f0d686f5c59499 golangci-lint-1.64.6-linux-mips64le.deb
57cea4538894558cb8c956d6b69c5509e4304546abe4a467478fc9ada0c736d6 golangci-lint-1.64.6-linux-mips64le.rpm
bc030977d44535b2152fddb2732f056d193c043992fe638ddecea21a40ef09fe golangci-lint-1.64.6-linux-mips64le.tar.gz
1ceb4e492efa527d246c61798c581f49113756fab8c39bb3eefdb1fa97041f92 golangci-lint-1.64.6-linux-ppc64le.deb
454e1c2b3583a8644e0c33a970fb4ce35b8f11848e1a770d9095d99d159ad0ab golangci-lint-1.64.6-linux-ppc64le.rpm
fddf30d35923eb6c7bb57520d8645768f802bf86c4cbf5a3a13049efb9e71b82 golangci-lint-1.64.6-linux-ppc64le.tar.gz
bd75f0dd6f65bee5821c433803b28f3c427ef3582764c3d0e7f7fae1c9d468b6 golangci-lint-1.64.6-linux-riscv64.deb
58457207c225cbd5340c8dcb75d9a44aa890d604e28464115a3a9762febaed04 golangci-lint-1.64.6-linux-riscv64.rpm
82639518a613a6705b7e2de5b28c278e875d782a5c97e9c1b2ac10b4ecd7852f golangci-lint-1.64.6-linux-riscv64.tar.gz
131d69204f8ced200b1437731e987cc886edac30dc87e6e1dcbd4f833d351713 golangci-lint-1.64.6-linux-s390x.deb
ca6caf28ca7a1df7cff720f8fd6ac4b8f2eac10c8cbfe7d2eade70876aded570 golangci-lint-1.64.6-linux-s390x.rpm
ed966d28a6512bc2b1120029a9f78ed77f2015e330b589a731d67d59be30b0b1 golangci-lint-1.64.6-linux-s390x.tar.gz
b181132b41943fc77ace7f9f5523085d12d3613f27774a0e35e17dd5e65ba589 golangci-lint-1.64.6-netbsd-386.tar.gz
f7b81c426006e3c699dc8665797a462aecba8c2cd23ac4971472e55184d95bc9 golangci-lint-1.64.6-netbsd-amd64.tar.gz
663d6fb4c3bef57b8aacdb1b1a4634075e55d294ed170724b443374860897ca6 golangci-lint-1.64.6-netbsd-arm64.tar.gz
8c7a76ee822568cc19352bbb9b2b0863dc5e185eb07782adbbaf648afd02ebcd golangci-lint-1.64.6-netbsd-armv6.tar.gz
0ce26d56ce78e516529e087118ba3f1bcd71d311b4c5b2bde6ec24a6e8966d85 golangci-lint-1.64.6-netbsd-armv7.tar.gz
135d5d998168683f46e4fab308cef5aa3c282025b7de6b258f976aadfb820db7 golangci-lint-1.64.6-source.tar.gz
ccdb0cc249531a205304a76308cfa7cda830083d083d557884416a2461148263 golangci-lint-1.64.6-windows-386.zip
2d88f282e08e1853c70bc7c914b5f58beaa6509903d56098aeb9bc3df30ea9d5 golangci-lint-1.64.6-windows-amd64.zip
6a3c6e7afc6916392679d7d6b95ac239827644e3e50ec4e8ca6ab1e4e6db65fe golangci-lint-1.64.6-windows-arm64.zip
9c9c1ef9687651566987f93e76252f7526c386901d7d8aeee044ca88115da4b1 golangci-lint-1.64.6-windows-armv6.zip
4f0df114155791799dfde8cd8cb6307fecfb17fed70b44205486ec925e2f39cf golangci-lint-1.64.6-windows-armv7.zip
# https://github.com/golangci/golangci-lint/releases/download/v2.0.2/
a88cbdc86b483fe44e90bf2dcc3fec2af8c754116e6edf0aa6592cac5baa7a0e golangci-lint-2.0.2-darwin-amd64.tar.gz
664550e7954f5f4451aae99b4f7382c1a47039c66f39ca605f5d9af1a0d32b49 golangci-lint-2.0.2-darwin-arm64.tar.gz
bda0f0f27d300502faceda8428834a76ca25986f6d9fc2bd41d201c3ed73f08e golangci-lint-2.0.2-freebsd-386.tar.gz
1cbd0c7ade3fb027d61d38a646ec1b51be5846952b4b04a5330e7f4687f2270c golangci-lint-2.0.2-freebsd-amd64.tar.gz
1e828a597726198b2e35acdbcc5f3aad85244d79846d2d2bdb05241c5a535f9e golangci-lint-2.0.2-freebsd-armv6.tar.gz
848b49315dc5cddd0c9ce35e96ab33d584db0ea8fb57bcbf9784f1622bec0269 golangci-lint-2.0.2-freebsd-armv7.tar.gz
cabf9a6beab574c7f98581eb237919e580024759e3cdc05c4d516b044dce6770 golangci-lint-2.0.2-illumos-amd64.tar.gz
2fde80d15ed6527791f106d606120620e913c3a663c90a8596861d0a4461169e golangci-lint-2.0.2-linux-386.deb
804bc6e350a8c613aaa0a33d8d45414a80157b0ba1b2c2335ac859f85ad98ebd golangci-lint-2.0.2-linux-386.rpm
e64beb72fecf581e57d88ae3adb1c9d4bf022694b6bd92e3c8e460910bbdc37d golangci-lint-2.0.2-linux-386.tar.gz
9c55aed174d7a52bb1d4006b36e7edee9023631f6b814a80cb39c9860d6f75c3 golangci-lint-2.0.2-linux-amd64.deb
c55a2ef741a687b4c679696931f7fd4a467babd64c9457cf17bb9632fd1cecd1 golangci-lint-2.0.2-linux-amd64.rpm
89cc8a7810dc63b9a37900da03e37c3601caf46d42265d774e0f1a5d883d53e2 golangci-lint-2.0.2-linux-amd64.tar.gz
a3e78583c4e7ea1b63e82559f126bb3a5b12788676f158526752d53e67824b99 golangci-lint-2.0.2-linux-arm64.deb
bd5dd52b5c9f18aa7a2904eda9a9f91c628e98623fe70b7afcbb847e2de84422 golangci-lint-2.0.2-linux-arm64.rpm
789d5b91219ac68c2336f77d41cd7e33a910420594780f455893f8453d09595b golangci-lint-2.0.2-linux-arm64.tar.gz
534cd4c464a66178714ed68152c1ed7aa73e5700bf409e4ed1a8363adf96afca golangci-lint-2.0.2-linux-armv6.deb
cf7d02905a5fc80b96c9a64621693b4cc7337b1ce29986c19fd72608dafe66c5 golangci-lint-2.0.2-linux-armv6.rpm
a0d81cb527d8fe878377f2356b5773e219b0b91832a6b59e7b9bcf9a90fe0b0e golangci-lint-2.0.2-linux-armv6.tar.gz
dedd5be7fff8cba8fe15b658a59347ea90d7d02a9fff87f09c7687e6da05a8b6 golangci-lint-2.0.2-linux-armv7.deb
85521b6f3ad2f5a2bc9bfe14b9b08623f764964048f75ed6dfcfaf8eb7d57cc1 golangci-lint-2.0.2-linux-armv7.rpm
96471046c7780dda4ea680f65e92c2ef56ff58d40bcffaf6cfe9d6d48e3c27aa golangci-lint-2.0.2-linux-armv7.tar.gz
815d914a7738e4362466b2d11004e8618b696b49e8ace13df2c2b25f28fb1e17 golangci-lint-2.0.2-linux-loong64.deb
f16381e3d8a0f011b95e086d83d620248432b915d01f4beab4d29cfe4dc388b0 golangci-lint-2.0.2-linux-loong64.rpm
1bd8d7714f9c92db6a0f23bae89f39c85ba047bec8eeb42b8748d30ae3228d18 golangci-lint-2.0.2-linux-loong64.tar.gz
ea6e9d4aabb526aa298e47e8b026d8893d918c5eb919ba0ab403e315def74cc5 golangci-lint-2.0.2-linux-mips64.deb
519d8d53af83fdc9c25cc3fba8b663331ac22ef68131d4b0084cb6f425b6f79a golangci-lint-2.0.2-linux-mips64.rpm
80d655a0a1ac1b19dcef4b58fa2a7dadb646cc50ad08d460b5c53cdb421165e4 golangci-lint-2.0.2-linux-mips64.tar.gz
aa0e75384bb482c865d4dfc95d23ceb25666bf20461b67a832f0eea6670312ec golangci-lint-2.0.2-linux-mips64le.deb
f2a8b500fb69bdea1b01df6267aaa5218fa4a58aeb781c1a20d0d802fe465a52 golangci-lint-2.0.2-linux-mips64le.rpm
e66a0c0c9a275f02d27a7caa9576112622306f001d73dfc082cf1ae446fc1242 golangci-lint-2.0.2-linux-mips64le.tar.gz
e85ad51aac6428be2d8a37000d053697371a538a5bcbc1644caa7c5e77f6d0af golangci-lint-2.0.2-linux-ppc64le.deb
906798365eac1944af2a9b9a303e6fd49ec9043307bc681b7a96277f7f8beea5 golangci-lint-2.0.2-linux-ppc64le.rpm
f7f1a271b0af274d6c9ce000f5dc6e1fb194350c67bcc62494f96f791882ba92 golangci-lint-2.0.2-linux-ppc64le.tar.gz
eea8bf643a42bf05de9780530db22923e5ab0d588f0e173594dc6518f2a25d2a golangci-lint-2.0.2-linux-riscv64.deb
4ff40f9fe2954400836e2a011ba4744d00ffab5068a51368552dfce6aba3b81b golangci-lint-2.0.2-linux-riscv64.rpm
531d8f225866674977d630afbf0533eb02f9bec607fb13895f7a2cd7b2e0a648 golangci-lint-2.0.2-linux-riscv64.tar.gz
6f827647046c603f40d97ea5aadc6f48cd0bb5d19f7a3d56500c3b833d2a0342 golangci-lint-2.0.2-linux-s390x.deb
387a090e9576d19ca86aac738172e58e07c19f2784a13bb387f4f0d75fb9c8d3 golangci-lint-2.0.2-linux-s390x.rpm
57de1fb7722a9feb2d11ed0a007a93959d05b9db5929a392abc222e30012467e golangci-lint-2.0.2-linux-s390x.tar.gz
ed95e0492ea86bf79eb661f0334474b2a4255093685ff587eccd797c5a54db7e golangci-lint-2.0.2-netbsd-386.tar.gz
eab81d729778166415d349a80e568b2f2b3a781745a9be3212a92abb1e732daf golangci-lint-2.0.2-netbsd-amd64.tar.gz
d20add73f7c2de2c3b01ed4fd7b63ffcf0a6597d5ea228d1699e92339a3cd047 golangci-lint-2.0.2-netbsd-arm64.tar.gz
4e4f44e6057879cd62424ff1800a767d25a595c0e91d6d48809eea9186b4c739 golangci-lint-2.0.2-netbsd-armv6.tar.gz
51ec17b16d8743ae4098a0171f04f0ed4d64561e3051b982778b0e6c306a1b03 golangci-lint-2.0.2-netbsd-armv7.tar.gz
5482cf27b93fae1765c70ee2a95d4074d038e9dee61bdd61d017ce8893d3a4a8 golangci-lint-2.0.2-source.tar.gz
a35d8fdf3e14079a10880dbbb7586b46faec89be96f086b244b3e565aac80313 golangci-lint-2.0.2-windows-386.zip
fe4b946cc01366b989001215687003a9c4a7098589921f75e6228d6d8cffc15c golangci-lint-2.0.2-windows-amd64.zip
646bd9250ef8c771d85cd22fe8e6f2397ae39599179755e3bbfa9ef97ad44090 golangci-lint-2.0.2-windows-arm64.zip
ce1dc0bad6f8a61d64e6b3779eeb773479c175125d6f686b0e67ef9c8432d16e golangci-lint-2.0.2-windows-armv6.zip
92684a48faabe792b11ac27ca8b25551eff940b0a1e84ad7244e98b4994962db golangci-lint-2.0.2-windows-armv7.zip
# This is the builder on PPA that will build Go itself (inception-y), don't modify!
#

View file

@ -43,6 +43,7 @@ func main() {
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
utils.BeaconCheckpointFileFlag,
//TODO datadir for optional permanent database
utils.MainnetFlag,
utils.SepoliaFlag,

View file

@ -1,6 +1,6 @@
# Clef
Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and asks for permission to sign the content. If the users grants the signing request, Clef will send the signature back to the DApp.
Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and ask for permission to sign the content. If the user grants the signing request, Clef will send the signature back to the DApp.
This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronized with the chain, or is a node that has no built-in (or limited) account management.
@ -123,7 +123,7 @@ The External API is **untrusted**: it does not accept credentials, nor does it e
Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API.
An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
An example (insecure) proof-of-concept has been implemented in `pythonsigner.py`.
The model is as follows:
@ -335,7 +335,7 @@ Bash example:
#### Arguments
- content type [string]: type of signed data
- `text/validator`: hex data with custom validator defined in a contract
- `text/validator`: hex data with a custom validator defined in a contract
- `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
- `text/plain`: simple hex data validated by `account_ecRecover`
- account [address]: account to sign with

View file

@ -2,7 +2,7 @@
The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
It enables usecases like the following:
It enables use cases like the following:
* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`

View file

@ -59,8 +59,9 @@ func (s *Suite) AllTests() []utesting.Test {
}
}
// TestPing sends PING and expects a PONG response.
func (s *Suite) TestPing(t *utesting.T) {
t.Log(`This test is just a sanity check. It sends PING and expects a PONG response.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -85,9 +86,10 @@ func checkPong(t *utesting.T, pong *v5wire.Pong, ping *v5wire.Ping, c net.Packet
}
}
// TestPingLargeRequestID sends PING with a 9-byte request ID, which isn't allowed by the spec.
// The remote node should not respond.
func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
t.Log(`This test sends PING with a 9-byte request ID, which isn't allowed by the spec.
The remote node should not respond.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -104,10 +106,11 @@ func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
}
}
// TestPingMultiIP establishes a session from one IP as usual. The session is then reused
// on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
// the attempt from a different IP.
func (s *Suite) TestPingMultiIP(t *utesting.T) {
t.Log(`This test establishes a session from one IP as usual. The session is then reused
on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
the attempt from a different IP.`)
conn, l1, l2 := s.listen2(t)
defer conn.close()
@ -120,6 +123,7 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
checkPong(t, resp.(*v5wire.Pong), ping, l1)
// Send on l2. This reuses the session because there is only one codec.
t.Log("sending ping from alternate IP", l2.LocalAddr())
ping2 := &v5wire.Ping{ReqID: conn.nextReqID()}
conn.write(l2, ping2, nil)
switch resp := conn.read(l2).(type) {
@ -158,6 +162,10 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
// packet instead of a handshake message packet. The remote node should respond with
// another WHOAREYOU challenge for the second packet.
func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message
packet instead of a handshake message packet. The remote node should respond with
another WHOAREYOU challenge for the second packet.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -181,8 +189,10 @@ func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
}
}
// TestTalkRequest sends TALKREQ and expects an empty TALKRESP response.
func (s *Suite) TestTalkRequest(t *utesting.T) {
t.Log(`This test sends some examples of TALKREQ with a protocol-id of "test-protocol"
and expects an empty TALKRESP response.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -202,6 +212,7 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
}
// Empty request ID.
t.Log("sending TALKREQ with empty request-id")
resp = conn.reqresp(l1, &v5wire.TalkRequest{Protocol: "test-protocol"})
switch resp := resp.(type) {
case *v5wire.TalkResponse:
@ -216,8 +227,9 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
}
}
// TestFindnodeZeroDistance checks that the remote node returns itself for FINDNODE with distance zero.
func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
t.Log(`This test checks that the remote node returns itself for FINDNODE with distance zero.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -233,9 +245,11 @@ func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
}
}
// TestFindnodeResults pings the node under test from multiple nodes. After waiting for them to be
// accepted into the remote table, the test checks that they are returned by FINDNODE.
func (s *Suite) TestFindnodeResults(t *utesting.T) {
t.Log(`This test pings the node under test from multiple other endpoints and node identities
(the 'bystanders'). After waiting for them to be accepted into the remote table, the test checks
that they are returned by FINDNODE.`)
// Create bystanders.
nodes := make([]*bystander, 5)
added := make(chan enode.ID, len(nodes))
@ -273,6 +287,7 @@ func (s *Suite) TestFindnodeResults(t *utesting.T) {
}
// Send FINDNODE for all distances.
t.Log("requesting nodes")
conn, l1 := s.listen1(t)
defer conn.close()
foundNodes, err := conn.findnode(l1, dists)

View file

@ -16,7 +16,7 @@ which can
1. Take a prestate, including
- Accounts,
- Block context information,
- Previous blockshashes (*optional)
- Previous block hashes (*optional)
2. Apply a set of transactions,
3. Apply a mining-reward (*optional),
4. And generate a post-state, including

View file

@ -22,7 +22,7 @@ import "regexp"
// within its filename by the execution spec test (EEST).
type testMetadata struct {
fork string
module string // which python module gnerated the test, e.g. eip7702
module string // which python module generated the test, e.g. eip7702
file string // exact file the test came from, e.g. test_gas.py
function string // func that created the test, e.g. test_valid_mcopy_operations
parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs

View file

@ -457,7 +457,7 @@ func importHistory(ctx *cli.Context) error {
network = networks[0]
}
if err := utils.ImportHistory(chain, db, dir, network); err != nil {
if err := utils.ImportHistory(chain, dir, network); err != nil {
return err
}
fmt.Printf("Import done in %v\n", time.Since(start))

View file

@ -157,6 +157,7 @@ var (
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
utils.BeaconCheckpointFileFlag,
}, utils.NetworkFlags, utils.DatabaseFlags)
rpcFlags = []cli.Flag{

View file

@ -246,8 +246,9 @@ func readList(filename string) ([]string, error) {
}
// ImportHistory imports Era1 files containing historical block information,
// starting from genesis.
func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
// starting from genesis. The assumption is held that the provided chain
// segment in Era1 file should all be canonical and verified.
func ImportHistory(chain *core.BlockChain, dir string, network string) error {
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
return errors.New("history import only supported when starting from genesis")
}
@ -308,11 +309,6 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
if err != nil {
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
}
if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start); err != nil {
return fmt.Errorf("error inserting header %d: %w", it.Number(), err)
} else if status != core.CanonStatTy {
return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status)
}
if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil {
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
}

View file

@ -342,6 +342,11 @@ var (
Usage: "Beacon chain weak subjectivity checkpoint block hash",
Category: flags.BeaconCategory,
}
BeaconCheckpointFileFlag = &cli.StringFlag{
Name: "beacon.checkpoint.file",
Usage: "Beacon chain weak subjectivity checkpoint import/export file",
Category: flags.BeaconCategory,
}
BlsyncApiFlag = &cli.StringFlag{
Name: "blsync.engine.api",
Usage: "Target EL engine API URL",
@ -1890,7 +1895,7 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
if !ctx.IsSet(BeaconGenesisTimeFlag.Name) {
Fatalf("Custom beacon chain config is specified but genesis time is missing")
}
if !ctx.IsSet(BeaconCheckpointFlag.Name) {
if !ctx.IsSet(BeaconCheckpointFlag.Name) && !ctx.IsSet(BeaconCheckpointFileFlag.Name) {
Fatalf("Custom beacon chain config is specified but checkpoint is missing")
}
config.ChainConfig = bparams.ChainConfig{
@ -1915,13 +1920,28 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
}
}
// Checkpoint is required with custom chain config and is optional with pre-defined config
if ctx.IsSet(BeaconCheckpointFlag.Name) {
if c, err := hexutil.Decode(ctx.String(BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 {
copy(config.Checkpoint[:len(c)], c)
} else {
Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(BeaconCheckpointFlag.Name), "error", err)
// If both checkpoint block hash and checkpoint file are specified then the
// client is initialized with the specified block hash and new checkpoints
// are saved to the specified file.
if ctx.IsSet(BeaconCheckpointFileFlag.Name) {
if _, err := config.SetCheckpointFile(ctx.String(BeaconCheckpointFileFlag.Name)); err != nil {
Fatalf("Could not load beacon checkpoint file", "beacon.checkpoint.file", ctx.String(BeaconCheckpointFileFlag.Name), "error", err)
}
}
if ctx.IsSet(BeaconCheckpointFlag.Name) {
hex := ctx.String(BeaconCheckpointFlag.Name)
c, err := hexutil.Decode(hex)
if err != nil {
Fatalf("Invalid hex string", "beacon.checkpoint", hex, "error", err)
}
if len(c) != 32 {
Fatalf("Invalid hex string length", "beacon.checkpoint", hex, "length", len(c))
}
copy(config.Checkpoint[:len(c)], c)
}
if config.Checkpoint == (common.Hash{}) {
Fatalf("Beacon checkpoint not specified")
}
config.Apis = ctx.StringSlice(BeaconApiFlag.Name)
if config.Apis == nil {
Fatalf("Beacon node light client API URL not specified")

View file

@ -171,7 +171,7 @@ func TestHistoryImportAndExport(t *testing.T) {
if err != nil {
t.Fatalf("unable to initialize chain: %v", err)
}
if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil {
if err := ImportHistory(imported, dir, "mainnet"); err != nil {
t.Fatalf("failed to import chain: %v", err)
}
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {

View file

@ -183,7 +183,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
if !disk {
db = rawdb.NewMemoryDatabase()
} else {
pdb, err := pebble.New(b.TempDir(), 128, 128, "", false)
pdb, err := pebble.New(b.TempDir(), 128, 128, "", false, true)
if err != nil {
b.Fatalf("cannot create temporary database: %v", err)
}
@ -303,7 +303,7 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin
func benchWriteChain(b *testing.B, full bool, count uint64) {
genesis := &Genesis{Config: params.AllEthashProtocolChanges}
for i := 0; i < b.N; i++ {
pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false)
pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false, true)
if err != nil {
b.Fatalf("error opening database: %v", err)
}
@ -316,7 +316,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) {
func benchReadChain(b *testing.B, full bool, count uint64) {
dir := b.TempDir()
pdb, err := pebble.New(dir, 1024, 128, "", false)
pdb, err := pebble.New(dir, 1024, 128, "", false, true)
if err != nil {
b.Fatalf("error opening database: %v", err)
}
@ -332,7 +332,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
pdb, err = pebble.New(dir, 1024, 128, "", false)
pdb, err = pebble.New(dir, 1024, 128, "", false, true)
if err != nil {
b.Fatalf("error opening database: %v", err)
}

View file

@ -24,6 +24,7 @@ import (
"math/big"
"runtime"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
@ -157,7 +158,8 @@ type CacheConfig struct {
// This defines the cutoff block for history expiry.
// Blocks before this number may be unavailable in the chain database.
HistoryPruningCutoff uint64
HistoryPruningCutoffNumber uint64
HistoryPruningCutoffHash common.Hash
}
// triedbConfig derives the configures for trie database.
@ -262,7 +264,6 @@ type BlockChain struct {
txLookupLock sync.RWMutex
txLookupCache *lru.Cache[common.Hash, txLookup]
wg sync.WaitGroup
quit chan struct{} // shutdown signal, closed in Stop.
stopping atomic.Bool // false if chain is running, true when stopped
procInterrupt atomic.Bool // interrupt signaler for block processing
@ -333,10 +334,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
bc.processor = NewStateProcessor(chainConfig, bc.hc)
genesisHeader := bc.GetHeaderByNumber(0)
bc.genesisBlock = types.NewBlockWithHeader(genesisHeader)
if bc.genesisBlock == nil {
if genesisHeader == nil {
return nil, ErrNoGenesis
}
bc.genesisBlock = types.NewBlockWithHeader(genesisHeader)
bc.currentBlock.Store(nil)
bc.currentSnapBlock.Store(nil)
@ -1110,7 +1111,6 @@ func (bc *BlockChain) stopWithoutSaving() {
// the mutex should become available quickly. It cannot be taken again after Close has
// returned.
bc.chainmu.Close()
bc.wg.Wait()
}
// Stop stops the blockchain service. If any imports are currently in progress
@ -1197,79 +1197,64 @@ const (
SideStatTy
)
// InsertReceiptChain attempts to complete an already existing header chain with
// transaction and receipt data.
// InsertReceiptChain inserts a batch of blocks along with their receipts into
// the database. Unlike InsertChain, this function does not verify the state root
// in the blocks. It is used exclusively for snap sync. All the inserted blocks
// will be regarded as canonical, chain reorg is not supported.
//
// The optional ancientLimit can also be specified and chain segment before that
// will be directly stored in the ancient, getting rid of the chain migration.
func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) {
// We don't require the chainMu here since we want to maximize the
// concurrency of header insertion and receipt insertion.
bc.wg.Add(1)
defer bc.wg.Done()
// Verify the supplied headers before insertion without lock
var headers []*types.Header
for _, block := range blockChain {
headers = append(headers, block.Header())
var (
ancientBlocks, liveBlocks types.Blocks
ancientReceipts, liveReceipts []types.Receipts
)
// Do a sanity check that the provided chain is actually ordered and linked
for i, block := range blockChain {
if i != 0 {
prev := blockChain[i-1]
if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
log.Error("Non contiguous receipt insert",
"number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(),
"prevnumber", prev.Number(), "prevhash", prev.Hash())
return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])",
i-1, prev.NumberU64(), prev.Hash().Bytes()[:4],
i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
}
}
if block.NumberU64() <= ancientLimit {
ancientBlocks, ancientReceipts = append(ancientBlocks, block), append(ancientReceipts, receiptChain[i])
} else {
liveBlocks, liveReceipts = append(liveBlocks, block), append(liveReceipts, receiptChain[i])
}
// Here we also validate that blob transactions in the block do not contain a sidecar.
// While the sidecar does not affect the block hash / tx hash, sending blobs within a block is not allowed.
// Here we also validate that blob transactions in the block do not
// contain a sidecar. While the sidecar does not affect the block hash
// or tx hash, sending blobs within a block is not allowed.
for txIndex, tx := range block.Transactions() {
if tx.Type() == types.BlobTxType && tx.BlobTxSidecar() != nil {
return 0, fmt.Errorf("block #%d contains unexpected blob sidecar in tx at index %d", block.NumberU64(), txIndex)
}
}
}
if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
return n, err
}
// Hold the mutation lock
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
var (
stats = struct{ processed, ignored int32 }{}
start = time.Now()
size = int64(0)
)
// updateHead updates the head snap sync block if the inserted blocks are better
// and returns an indicator whether the inserted blocks are canonical.
updateHead := func(head *types.Block) bool {
if !bc.chainmu.TryLock() {
return false
// updateHead updates the head header and head snap block flags.
updateHead := func(header *types.Header) error {
batch := bc.db.NewBatch()
hash := header.Hash()
rawdb.WriteHeadHeaderHash(batch, hash)
rawdb.WriteHeadFastBlockHash(batch, hash)
if err := batch.Write(); err != nil {
return err
}
defer bc.chainmu.Unlock()
// Rewind may have occurred, skip in that case.
if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 {
rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
bc.currentSnapBlock.Store(head.Header())
headFastBlockGauge.Update(int64(head.NumberU64()))
return true
}
return false
bc.hc.currentHeader.Store(header)
bc.currentSnapBlock.Store(header)
headHeaderGauge.Update(header.Number.Int64())
headFastBlockGauge.Update(header.Number.Int64())
return nil
}
// writeAncient writes blockchain and corresponding receipt chain into ancient store.
//
// this function only accepts canonical chain data. All side chain will be reverted
// eventually.
writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
first := blockChain[0]
last := blockChain[len(blockChain)-1]
// Ensure genesis is in ancients.
if first.NumberU64() == 1 {
// Ensure genesis is in the ancient store
if blockChain[0].NumberU64() == 1 {
if frozen, _ := bc.db.Ancients(); frozen == 0 {
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
if err != nil {
@ -1280,12 +1265,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
log.Info("Wrote genesis to ancients")
}
}
// Before writing the blocks to the ancients, we need to ensure that
// they correspond to the what the headerchain 'expects'.
// We only check the last block/header, since it's a contiguous chain.
if !bc.HasHeader(last.Hash(), last.NumberU64()) {
return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4])
}
// Write all chain data to ancients.
writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain)
if err != nil {
@ -1298,44 +1277,28 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
if err := bc.db.Sync(); err != nil {
return 0, err
}
// Update the current snap block because all block data is now present in DB.
previousSnapBlock := bc.CurrentSnapBlock().Number.Uint64()
if !updateHead(blockChain[len(blockChain)-1]) {
// We end up here if the header chain has reorg'ed, and the blocks/receipts
// don't match the canonical chain.
if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
log.Error("Can't truncate ancient store after failed insert", "err", err)
}
return 0, errSideChainReceipts
}
// Delete block data from the main database.
var (
batch = bc.db.NewBatch()
canonHashes = make(map[common.Hash]struct{}, len(blockChain))
)
// Write hash to number mappings
batch := bc.db.NewBatch()
for _, block := range blockChain {
canonHashes[block.Hash()] = struct{}{}
if block.NumberU64() == 0 {
continue
}
rawdb.DeleteCanonicalHash(batch, block.NumberU64())
rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64())
}
// Delete side chain hash-to-number mappings.
for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) {
if _, canon := canonHashes[nh.Hash]; !canon {
rawdb.DeleteHeader(batch, nh.Hash, nh.Number)
}
rawdb.WriteHeaderNumber(batch, block.Hash(), block.NumberU64())
}
if err := batch.Write(); err != nil {
return 0, err
}
// Update the current snap block because all block data is now present in DB.
if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
return 0, err
}
stats.processed += int32(len(blockChain))
return 0, nil
}
// writeLive writes blockchain and corresponding receipt chain into active store.
// writeLive writes the blockchain and corresponding receipt chain to the active store.
//
// Notably, in different snap sync cycles, the supplied chain may partially reorganize
// existing local chain segments (reorg around the chain tip). The reorganized part
// will be included in the provided chain segment, and stale canonical markers will be
// silently rewritten. Therefore, no explicit reorg logic is needed.
writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
var (
skipPresenceCheck = false
@ -1346,10 +1309,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
if bc.insertStopped() {
return 0, errInsertionInterrupted
}
// Short circuit if the owner header is unknown
if !bc.HasHeader(block.Hash(), block.NumberU64()) {
return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4])
}
if !skipPresenceCheck {
// Ignore if the entire data is already known
if bc.HasBlock(block.Hash(), block.NumberU64()) {
@ -1363,7 +1322,8 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
}
}
// Write all the data out into the database
rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body())
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
rawdb.WriteBlock(batch, block)
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
// Write everything belongs to the blocks into the database. So that
@ -1387,21 +1347,27 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
return 0, err
}
}
updateHead(blockChain[len(blockChain)-1])
if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
return 0, err
}
return 0, nil
}
// Write downloaded chain data and corresponding receipt chain data
if len(ancientBlocks) > 0 {
if n, err := writeAncient(ancientBlocks, ancientReceipts); err != nil {
// Split the supplied blocks into two groups, according to the
// given ancient limit.
index := sort.Search(len(blockChain), func(i int) bool {
return blockChain[i].NumberU64() >= ancientLimit
})
if index > 0 {
if n, err := writeAncient(blockChain[:index], receiptChain[:index]); err != nil {
if err == errInsertionInterrupted {
return 0, nil
}
return n, err
}
}
if len(liveBlocks) > 0 {
if n, err := writeLive(liveBlocks, liveReceipts); err != nil {
if index != len(blockChain) {
if n, err := writeLive(blockChain[index:], receiptChain[index:]); err != nil {
if err == errInsertionInterrupted {
return 0, nil
}
@ -1420,7 +1386,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
context = append(context, []interface{}{"ignored", stats.ignored}...)
}
log.Debug("Imported new block receipts", context...)
return 0, nil
}
@ -2505,15 +2470,83 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header) (int, error) {
if i, err := bc.hc.ValidateHeaderChain(chain); err != nil {
return i, err
}
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
_, err := bc.hc.InsertHeaderChain(chain, start)
return 0, err
}
// InsertHeadersBeforeCutoff inserts the given headers into the ancient store
// as they are claimed older than the configured chain cutoff point. All the
// inserted headers are regarded as canonical and chain reorg is not supported.
func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, error) {
if len(headers) == 0 {
return 0, nil
}
// TODO(rjl493456442): Headers before the configured cutoff have already
// been verified by the hash of cutoff header. Theoretically, header validation
// could be skipped here.
if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
return n, err
}
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
// Initialize the ancient store with genesis block if it's empty.
var (
frozen, _ = bc.db.Ancients()
first = headers[0].Number.Uint64()
)
if first == 1 && frozen == 0 {
_, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err
}
log.Info("Wrote genesis to ancient store")
} else if frozen != first {
return 0, fmt.Errorf("headers are gapped with the ancient store, first: %d, ancient: %d", first, frozen)
}
// Write headers to the ancient store, with block bodies and receipts set to nil
// to ensure consistency across tables in the freezer.
_, err := rawdb.WriteAncientHeaderChain(bc.db, headers)
if err != nil {
return 0, err
}
if err := bc.db.Sync(); err != nil {
return 0, err
}
// Write hash to number mappings
batch := bc.db.NewBatch()
for _, header := range headers {
rawdb.WriteHeaderNumber(batch, header.Hash(), header.Number.Uint64())
}
// Write head header and head snap block flags
last := headers[len(headers)-1]
rawdb.WriteHeadHeaderHash(batch, last.Hash())
rawdb.WriteHeadFastBlockHash(batch, last.Hash())
if err := batch.Write(); err != nil {
return 0, err
}
// Truncate the useless chain segment (zero bodies and receipts) in the
// ancient store.
if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil {
return 0, err
}
// Last step update all in-memory markers
bc.hc.currentHeader.Store(last)
bc.currentSnapBlock.Store(last)
headHeaderGauge.Update(last.Number.Int64())
headFastBlockGauge.Update(last.Number.Int64())
return 0, nil
}
// SetBlockValidatorAndProcessorForTesting sets the current validator and processor.
// This method can be used to force an invalid blockchain to be verified for tests.
// This method is unsafe and should only be used before block import starts.
@ -2533,9 +2566,3 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
return time.Duration(bc.flushInterval.Load())
}
// HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() uint64 {
return bc.cacheConfig.HistoryPruningCutoff
}

View file

@ -407,6 +407,12 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) {
return bc.txIndexer.txIndexProgress()
}
// HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) {
return bc.cacheConfig.HistoryPruningCutoffNumber, bc.cacheConfig.HistoryPruningCutoffHash
}
// TrieDB retrieves the low level trie database used for data storage.
func (bc *BlockChain) TrieDB() *triedb.Database {
return bc.triedb

View file

@ -1765,7 +1765,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
pdb, err := pebble.New(datadir, 0, 0, "", false)
pdb, err := pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
@ -1850,7 +1850,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
pdb, err = pebble.New(datadir, 0, 0, "", false)
pdb, err = pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
@ -1915,7 +1915,7 @@ func testIssue23496(t *testing.T, scheme string) {
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
pdb, err := pebble.New(datadir, 0, 0, "", false)
pdb, err := pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
@ -1973,7 +1973,7 @@ func testIssue23496(t *testing.T, scheme string) {
chain.stopWithoutSaving()
// Start a new blockchain back up and see where the repair leads us
pdb, err = pebble.New(datadir, 0, 0, "", false)
pdb, err = pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}

View file

@ -1969,7 +1969,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
pdb, err := pebble.New(datadir, 0, 0, "", false)
pdb, err := pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}

View file

@ -66,7 +66,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
datadir := t.TempDir()
ancient := filepath.Join(datadir, "ancient")
pdb, err := pebble.New(datadir, 0, 0, "", false)
pdb, err := pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
@ -257,7 +257,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
chain.triedb.Close()
// Start a new blockchain back up and see where the repair leads us
pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false)
pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}

View file

@ -733,13 +733,6 @@ func testFastVsFullChains(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -753,9 +746,6 @@ func testFastVsFullChains(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -880,13 +870,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -900,9 +883,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -916,6 +896,11 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
// Import the chain as a light node and ensure all pointers are updated
lightDb := makeDb()
defer lightDb.Close()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
if n, err := light.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
@ -1710,13 +1695,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
defer ancientDb.Close()
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -1741,82 +1719,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
}
}
// This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain.
func TestInsertReceiptChainRollback(t *testing.T) {
testInsertReceiptChainRollback(t, rawdb.HashScheme)
testInsertReceiptChainRollback(t, rawdb.PathScheme)
}
func testInsertReceiptChainRollback(t *testing.T, scheme string) {
// Generate forked chain. The returned BlockChain object is used to process the side chain blocks.
tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme)
if err != nil {
t.Fatal(err)
}
defer tmpChain.Stop()
// Get the side chain receipts.
if _, err := tmpChain.InsertChain(sideblocks); err != nil {
t.Fatal("processing side chain failed:", err)
}
t.Log("sidechain head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
sidechainReceipts := make([]types.Receipts, len(sideblocks))
for i, block := range sideblocks {
sidechainReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
}
// Get the canon chain receipts.
if _, err := tmpChain.InsertChain(canonblocks); err != nil {
t.Fatal("processing canon chain failed:", err)
}
t.Log("canon head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
canonReceipts := make([]types.Receipts, len(canonblocks))
for i, block := range canonblocks {
canonReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
}
// Set up a BlockChain that uses the ancient store.
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
defer ancientDb.Close()
ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancientChain.Stop()
// Import the canonical header chain.
canonHeaders := make([]*types.Header, len(canonblocks))
for i, block := range canonblocks {
canonHeaders[i] = block.Header()
}
if _, err = ancientChain.InsertHeaderChain(canonHeaders); err != nil {
t.Fatal("can't import canon headers:", err)
}
// Try to insert blocks/receipts of the side chain.
_, err = ancientChain.InsertReceiptChain(sideblocks, sidechainReceipts, uint64(len(sideblocks)))
if err == nil {
t.Fatal("expected error from InsertReceiptChain.")
}
if ancientChain.CurrentSnapBlock().Number.Uint64() != 0 {
t.Fatalf("failed to rollback ancient data, want %d, have %d", 0, ancientChain.CurrentSnapBlock().Number)
}
if frozen, err := ancientChain.db.Ancients(); err != nil || frozen != 1 {
t.Fatalf("failed to truncate ancient data, frozen index is %d", frozen)
}
// Insert blocks/receipts of the canonical chain.
_, err = ancientChain.InsertReceiptChain(canonblocks, canonReceipts, uint64(len(canonblocks)))
if err != nil {
t.Fatalf("can't import canon chain receipts: %v", err)
}
if ancientChain.CurrentSnapBlock().Number.Uint64() != canonblocks[len(canonblocks)-1].NumberU64() {
t.Fatalf("failed to insert ancient recept chain after rollback")
}
if frozen, _ := ancientChain.db.Ancients(); frozen != uint64(len(canonblocks))+1 {
t.Fatalf("wrong ancients count %d", frozen)
}
}
// Tests that importing a very large side fork, which is larger than the canon chain,
// but where the difficulty per block is kept low: this means that it will not
// overtake the 'canon' chain until after it's passed canon by about 200 blocks.
@ -2088,14 +1990,6 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
headers := make([]*types.Header, 0, len(blocks))
for _, block := range blocks {
headers = append(headers, block.Header())
}
_, err := chain.InsertHeaderChain(headers)
if err != nil {
return err
}
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
return err
}
@ -2262,14 +2156,6 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
headers := make([]*types.Header, 0, len(blocks))
for _, block := range blocks {
headers = append(headers, block.Header())
}
i, err := chain.InsertHeaderChain(headers)
if err != nil {
return fmt.Errorf("index %d: %w", i, err)
}
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
return err
}
@ -2605,7 +2491,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
datadir := t.TempDir()
ancient := path.Join(datadir, "ancient")
pdb, err := pebble.New(datadir, 0, 0, "", false)
pdb, err := pebble.New(datadir, 0, 0, "", false, true)
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
@ -4265,3 +4151,220 @@ func TestEIP7702(t *testing.T) {
t.Fatalf("addr2 storage wrong: expected %d, got %d", fortyTwo, actual)
}
}
// Tests the scenario that the synchronization target in snap sync has been changed
// with a chain reorg at the tip. In this case the reorg'd segment should be unmarked
// with canonical flags.
func TestChainReorgSnapSync(t *testing.T) {
testChainReorgSnapSync(t, 0)
testChainReorgSnapSync(t, 32)
testChainReorgSnapSync(t, gomath.MaxUint64)
}
func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
// Configure and generate a sample block chain
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000000000)
gspec = &Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{address: {Balance: funds}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
signer = types.LatestSigner(gspec.Config)
engine = beacon.New(ethash.NewFaker())
)
genDb, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 32, func(i int, block *BlockGen) {
block.SetCoinbase(common.Address{0x00})
// If the block number is multiple of 3, send a few bonus transactions to the miner
if i%3 == 2 {
for j := 0; j < i%4+1; j++ {
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
if err != nil {
panic(err)
}
block.AddTx(tx)
}
}
})
chainA, receiptsA := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 16, func(i int, gen *BlockGen) {
gen.SetCoinbase(common.Address{0: byte(0xa), 19: byte(i)})
})
chainB, receiptsB := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 20, func(i int, gen *BlockGen) {
gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
})
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
if n, err := chain.InsertReceiptChain(blocks, receipts, ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(chainA, receiptsA, ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// If the common ancestor is below the ancient limit, rewind the chain head.
// It's aligned with the behavior in the snap sync
ancestor := blocks[len(blocks)-1].NumberU64()
if ancestor < ancientLimit {
rawdb.WriteLastPivotNumber(db, ancestor)
chain.SetHead(ancestor)
}
if n, err := chain.InsertReceiptChain(chainB, receiptsB, ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
head := chain.CurrentSnapBlock()
if head.Hash() != chainB[len(chainB)-1].Hash() {
t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", head.Number, chainB[len(chainB)-1].Hash(), head.Hash())
}
// Iterate over all chain data components, and cross reference
for i := 0; i < len(blocks); i++ {
num, hash := blocks[i].NumberU64(), blocks[i].Hash()
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
}
for i := 0; i < len(chainA); i++ {
num, hash := chainA[i].NumberU64(), chainA[i].Hash()
header := chain.GetHeaderByNumber(num)
if header == nil {
continue
}
if header.Hash() == hash {
t.Errorf("block #%d: unexpected canonical header: %v", num, hash)
}
}
for i := 0; i < len(chainB); i++ {
num, hash := chainB[i].NumberU64(), chainB[i].Hash()
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
}
}
// Tests the scenario that all the inserted chain segment are with the configured
// chain cutoff point. In this case the chain segment before the cutoff should
// be persisted without the receipts and bodies; chain after should be persisted
// normally.
func TestInsertChainWithCutoff(t *testing.T) {
testInsertChainWithCutoff(t, 32, 32) // cutoff = 32, ancientLimit = 32
testInsertChainWithCutoff(t, 32, 64) // cutoff = 32, ancientLimit = 64 (entire chain in ancient)
testInsertChainWithCutoff(t, 32, 65) // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live)
}
func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) {
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
// Configure and generate a sample block chain
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000000000)
gspec = &Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{address: {Balance: funds}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
signer = types.LatestSigner(gspec.Config)
engine = beacon.New(ethash.NewFaker())
)
_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(2*cutoff), func(i int, block *BlockGen) {
block.SetCoinbase(common.Address{0x00})
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
if err != nil {
panic(err)
}
block.AddTx(tx)
})
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
defer db.Close()
cutoffBlock := blocks[cutoff-1]
config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
config.HistoryPruningCutoffNumber = cutoffBlock.NumberU64()
config.HistoryPruningCutoffHash = cutoffBlock.Hash()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
var (
headersBefore []*types.Header
blocksAfter []*types.Block
receiptsAfter []types.Receipts
)
for i, b := range blocks {
if b.NumberU64() < cutoffBlock.NumberU64() {
headersBefore = append(headersBefore, b.Header())
} else {
blocksAfter = append(blocksAfter, b)
receiptsAfter = append(receiptsAfter, receipts[i])
}
}
if n, err := chain.InsertHeadersBeforeCutoff(headersBefore); err != nil {
t.Fatalf("failed to insert headers before cutoff %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(blocksAfter, receiptsAfter, ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
headSnap := chain.CurrentSnapBlock()
if headSnap.Hash() != blocks[len(blocks)-1].Hash() {
t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", headSnap.Number, blocks[len(blocks)-1].Hash(), headSnap.Hash())
}
headHeader := chain.CurrentHeader()
if headHeader.Hash() != blocks[len(blocks)-1].Hash() {
t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash())
}
headBlock := chain.CurrentBlock()
if headBlock.Hash() != gspec.ToBlock().Hash() {
t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, gspec.ToBlock().Hash(), headBlock.Hash())
}
// Iterate over all chain data components, and cross reference
for i := 0; i < len(blocks); i++ {
num, hash := blocks[i].NumberU64(), blocks[i].Hash()
// Canonical headers should be visible regardless of cutoff
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
tail, err := db.Tail()
if err != nil {
t.Fatalf("Failed to get chain tail, %v", err)
}
if tail != cutoffBlock.NumberU64() {
t.Fatalf("Unexpected chain tail, want: %d, got: %d", cutoffBlock.NumberU64(), tail)
}
// Block bodies and receipts before the cutoff should be non-existent
if num < cutoffBlock.NumberU64() {
body := chain.GetBody(hash)
if body != nil {
t.Fatalf("Unexpected block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
receipts := chain.GetReceiptsByHash(hash)
if receipts != nil {
t.Fatalf("Unexpected block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
} else {
body := chain.GetBody(hash)
if body == nil || len(body.Transactions) != 1 {
t.Fatalf("Missed block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
receipts := chain.GetReceiptsByHash(hash)
if receipts == nil || len(receipts) != 1 {
t.Fatalf("Missed block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
}
}
}

View file

@ -28,8 +28,6 @@ var (
// ErrNoGenesis is returned when there is no Genesis Block.
ErrNoGenesis = errors.New("genesis not found in chain")
errSideChainReceipts = errors.New("side blocks can't be accepted as ancient chain data")
)
// List of evm-call-message pre-checking errors. All state transition messages will

View file

@ -237,7 +237,8 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) {
}
// writeHeadersAndSetHead writes a batch of block headers and applies the last
// header as the chain head if the fork choicer says it's ok to update the chain.
// header as the chain head.
//
// Note: This method is not concurrent-safe with inserting blocks simultaneously
// into the chain, as side effects caused by reorganisations cannot be emulated
// without the real blocks. Hence, writing headers directly should only be done
@ -272,12 +273,14 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header) (*headerW
return result, nil
}
// ValidateHeaderChain verifies that the supplied header chain is contiguous
// and conforms to consensus rules.
func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header) (int, error) {
// Do a sanity check that the provided chain is actually ordered and linked
for i := 1; i < len(chain); i++ {
if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 {
hash := chain[i].Hash()
parentHash := chain[i-1].Hash()
hash, parentHash := chain[i].Hash(), chain[i-1].Hash()
// Chain broke ancestry, log a message (programming error) and skip insertion
log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", hash,
"parent", chain[i].ParentHash, "prevnumber", chain[i-1].Number, "prevhash", parentHash)
@ -302,7 +305,6 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header) (int, error) {
return i, err
}
}
return 0, nil
}

View file

@ -737,6 +737,30 @@ func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *type
return nil
}
// WriteAncientHeaderChain writes the supplied headers along with nil block
// bodies and receipts into the ancient store. It's supposed to be used for
// storing chain segment before the chain cutoff.
func WriteAncientHeaderChain(db ethdb.AncientWriter, headers []*types.Header) (int64, error) {
return db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
for _, header := range headers {
num := header.Number.Uint64()
if err := op.AppendRaw(ChainFreezerHashTable, num, header.Hash().Bytes()); err != nil {
return fmt.Errorf("can't add block %d hash: %v", num, err)
}
if err := op.Append(ChainFreezerHeaderTable, num, header); err != nil {
return fmt.Errorf("can't append block header %d: %v", num, err)
}
if err := op.AppendRaw(ChainFreezerBodiesTable, num, nil); err != nil {
return fmt.Errorf("can't append block body %d: %v", num, err)
}
if err := op.AppendRaw(ChainFreezerReceiptTable, num, nil); err != nil {
return fmt.Errorf("can't append block %d receipts: %v", num, err)
}
}
return nil
})
}
// DeleteBlock removes all block data associated with a hash.
func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
DeleteReceipts(db, hash, number)

View file

@ -464,6 +464,48 @@ func TestAncientStorage(t *testing.T) {
}
}
func TestWriteAncientHeaderChain(t *testing.T) {
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false)
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
defer db.Close()
// Create a test block
var headers []*types.Header
headers = append(headers, &types.Header{
Number: big.NewInt(0),
Extra: []byte("test block"),
UncleHash: types.EmptyUncleHash,
TxHash: types.EmptyTxsHash,
ReceiptHash: types.EmptyReceiptsHash,
})
headers = append(headers, &types.Header{
Number: big.NewInt(1),
Extra: []byte("test block"),
UncleHash: types.EmptyUncleHash,
TxHash: types.EmptyTxsHash,
ReceiptHash: types.EmptyReceiptsHash,
})
// Write and verify the header in the database
WriteAncientHeaderChain(db, headers)
for _, header := range headers {
if blob := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); len(blob) == 0 {
t.Fatalf("no header returned")
}
if h := ReadCanonicalHash(db, header.Number.Uint64()); h != header.Hash() {
t.Fatalf("no canonical hash returned")
}
if blob := ReadBodyRLP(db, header.Hash(), header.Number.Uint64()); len(blob) != 0 {
t.Fatalf("unexpected body returned")
}
if blob := ReadReceiptsRLP(db, header.Hash(), header.Number.Uint64()); len(blob) != 0 {
t.Fatalf("unexpected body returned")
}
}
}
func TestCanonicalHashIteration(t *testing.T) {
var cases = []struct {
from, to uint64

View file

@ -58,9 +58,10 @@ type txIndexer struct {
// newTxIndexer initializes the transaction indexer.
func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
cutoff, _ := chain.HistoryPruningCutoff()
indexer := &txIndexer{
limit: limit,
cutoff: chain.HistoryPruningCutoff(),
cutoff: cutoff,
db: chain.db,
progress: make(chan chan TxIndexProgress),
term: make(chan chan struct{}),

View file

@ -87,8 +87,9 @@ type blobTxMeta struct {
hash common.Hash // Transaction hash to maintain the lookup table
vhashes []common.Hash // Blob versioned hashes to maintain the lookup table
id uint64 // Storage ID in the pool's persistent store
size uint32 // Byte size in the pool's persistent store
id uint64 // Storage ID in the pool's persistent store
storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob
nonce uint64 // Needed to prioritize inclusion order within an account
costCap *uint256.Int // Needed to validate cumulative balance sufficiency
@ -108,19 +109,20 @@ type blobTxMeta struct {
// newBlobTxMeta retrieves the indexed metadata fields from a blob transaction
// and assembles a helper struct to track in memory.
func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta {
func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transaction) *blobTxMeta {
meta := &blobTxMeta{
hash: tx.Hash(),
vhashes: tx.BlobHashes(),
id: id,
size: size,
nonce: tx.Nonce(),
costCap: uint256.MustFromBig(tx.Cost()),
execTipCap: uint256.MustFromBig(tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()),
execGas: tx.Gas(),
blobGas: tx.BlobGas(),
hash: tx.Hash(),
vhashes: tx.BlobHashes(),
id: id,
storageSize: storageSize,
size: size,
nonce: tx.Nonce(),
costCap: uint256.MustFromBig(tx.Cost()),
execTipCap: uint256.MustFromBig(tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()),
execGas: tx.Gas(),
blobGas: tx.BlobGas(),
}
meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap)
meta.blobfeeJumps = dynamicFeeJumps(meta.blobFeeCap)
@ -480,7 +482,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
return errors.New("missing blob sidecar")
}
meta := newBlobTxMeta(id, size, tx)
meta := newBlobTxMeta(id, tx.Size(), size, tx)
if p.lookup.exists(meta.hash) {
// This path is only possible after a crash, where deleted items are not
// removed via the normal shutdown-startup procedure and thus may get
@ -507,7 +509,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
p.spent[sender] = new(uint256.Int).Add(p.spent[sender], meta.costCap)
p.lookup.track(meta)
p.stored += uint64(meta.size)
p.stored += uint64(meta.storageSize)
return nil
}
@ -539,7 +541,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
ids = append(ids, txs[i].id)
nonces = append(nonces, txs[i].nonce)
p.stored -= uint64(txs[i].size)
p.stored -= uint64(txs[i].storageSize)
p.lookup.untrack(txs[i])
// Included transactions blobs need to be moved to the limbo
@ -580,7 +582,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
nonces = append(nonces, txs[0].nonce)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[0].costCap)
p.stored -= uint64(txs[0].size)
p.stored -= uint64(txs[0].storageSize)
p.lookup.untrack(txs[0])
// Included transactions blobs need to be moved to the limbo
@ -636,7 +638,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
dropRepeatedMeter.Mark(1)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap)
p.stored -= uint64(txs[i].size)
p.stored -= uint64(txs[i].storageSize)
p.lookup.untrack(txs[i])
if err := p.store.Delete(id); err != nil {
@ -658,7 +660,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
nonces = append(nonces, txs[j].nonce)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[j].costCap)
p.stored -= uint64(txs[j].size)
p.stored -= uint64(txs[j].storageSize)
p.lookup.untrack(txs[j])
}
txs = txs[:i]
@ -696,7 +698,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
nonces = append(nonces, last.nonce)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap)
p.stored -= uint64(last.size)
p.stored -= uint64(last.storageSize)
p.lookup.untrack(last)
}
if len(txs) == 0 {
@ -736,7 +738,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
nonces = append(nonces, last.nonce)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap)
p.stored -= uint64(last.size)
p.stored -= uint64(last.storageSize)
p.lookup.untrack(last)
}
p.index[addr] = txs
@ -1002,7 +1004,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
}
// Update the indices and metrics
meta := newBlobTxMeta(id, p.store.Size(id), tx)
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx)
if _, ok := p.index[addr]; !ok {
if err := p.reserve(addr, true); err != nil {
log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err)
@ -1016,7 +1018,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
p.spent[addr] = new(uint256.Int).Add(p.spent[addr], meta.costCap)
}
p.lookup.track(meta)
p.stored += uint64(meta.size)
p.stored += uint64(meta.storageSize)
return nil
}
@ -1041,7 +1043,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) {
nonces = []uint64{tx.nonce}
)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap)
p.stored -= uint64(tx.size)
p.stored -= uint64(tx.storageSize)
p.lookup.untrack(tx)
txs[i] = nil
@ -1051,7 +1053,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) {
nonces = append(nonces, tx.nonce)
p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], tx.costCap)
p.stored -= uint64(tx.size)
p.stored -= uint64(tx.storageSize)
p.lookup.untrack(tx)
txs[i+1+j] = nil
}
@ -1236,6 +1238,25 @@ func (p *BlobPool) GetRLP(hash common.Hash) []byte {
return p.getRLP(hash)
}
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
//
// The size refers the length of the 'rlp encoding' of a blob transaction
// including the attached blobs.
func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.RLock()
defer p.lock.RUnlock()
size, ok := p.lookup.sizeOfTx(hash)
if !ok {
return nil
}
return &txpool.TxMetadata{
Type: types.BlobTxType,
Size: size,
}
}
// GetBlobs returns a number of blobs are proofs for the given versioned hashes.
// This is a utility method for the engine API, enabling consensus clients to
// retrieve blobs from the pools directly instead of the network.
@ -1375,7 +1396,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
if err != nil {
return err
}
meta := newBlobTxMeta(id, p.store.Size(id), tx)
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx)
var (
next = p.state.GetNonce(from)
@ -1403,7 +1424,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
p.lookup.untrack(prev)
p.lookup.track(meta)
p.stored += uint64(meta.size) - uint64(prev.size)
p.stored += uint64(meta.storageSize) - uint64(prev.storageSize)
} else {
// Transaction extends previously scheduled ones
p.index[from] = append(p.index[from], meta)
@ -1413,7 +1434,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
}
p.spent[from] = new(uint256.Int).Add(p.spent[from], meta.costCap)
p.lookup.track(meta)
p.stored += uint64(meta.size)
p.stored += uint64(meta.storageSize)
}
// Recompute the rolling eviction fields. In case of a replacement, this will
// recompute all subsequent fields. In case of an append, this will only do
@ -1500,7 +1521,7 @@ func (p *BlobPool) drop() {
p.index[from] = txs
p.spent[from] = new(uint256.Int).Sub(p.spent[from], drop.costCap)
}
p.stored -= uint64(drop.size)
p.stored -= uint64(drop.storageSize)
p.lookup.untrack(drop)
// Remove the transaction from the pool's eviction heap:

View file

@ -376,7 +376,7 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) {
var stored uint64
for _, txs := range pool.index {
for _, tx := range txs {
stored += uint64(tx.size)
stored += uint64(tx.storageSize)
}
}
if pool.stored != stored {
@ -1553,6 +1553,16 @@ func TestAdd(t *testing.T) {
if err := pool.add(signed); !errors.Is(err, add.err) {
t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err)
}
if add.err == nil {
size, exist := pool.lookup.sizeOfTx(signed.Hash())
if !exist {
t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j)
}
if size != signed.Size() {
t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v",
i, j, size, signed.Size())
}
}
verifyPoolInternals(t, pool)
}
verifyPoolInternals(t, pool)

View file

@ -146,7 +146,7 @@ func TestPriceHeapSorting(t *testing.T) {
)
index[addr] = []*blobTxMeta{{
id: uint64(j),
size: 128 * 1024,
storageSize: 128 * 1024,
nonce: 0,
execTipCap: execTip,
execFeeCap: execFee,
@ -205,7 +205,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) {
)
index[addr] = []*blobTxMeta{{
id: uint64(i),
size: 128 * 1024,
storageSize: 128 * 1024,
nonce: 0,
execTipCap: execTip,
execFeeCap: execFee,
@ -281,7 +281,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) {
)
index[addr] = []*blobTxMeta{{
id: uint64(i),
size: 128 * 1024,
storageSize: 128 * 1024,
nonce: 0,
execTipCap: execTip,
execFeeCap: execFee,
@ -312,7 +312,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) {
)
metas[i] = &blobTxMeta{
id: uint64(int(blobs) + i),
size: 128 * 1024,
storageSize: 128 * 1024,
nonce: 0,
execTipCap: execTip,
execFeeCap: execFee,

View file

@ -20,18 +20,24 @@ import (
"github.com/ethereum/go-ethereum/common"
)
type txMetadata struct {
id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included)
}
// lookup maps blob versioned hashes to transaction hashes that include them,
// and transaction hashes to billy entries that include them.
// transaction hashes to billy entries that include them, transaction hashes
// to the transaction size
type lookup struct {
blobIndex map[common.Hash]map[common.Hash]struct{}
txIndex map[common.Hash]uint64
txIndex map[common.Hash]*txMetadata
}
// newLookup creates a new index for tracking blob to tx; and tx to billy mappings.
func newLookup() *lookup {
return &lookup{
blobIndex: make(map[common.Hash]map[common.Hash]struct{}),
txIndex: make(map[common.Hash]uint64),
txIndex: make(map[common.Hash]*txMetadata),
}
}
@ -43,8 +49,11 @@ func (l *lookup) exists(txhash common.Hash) bool {
// storeidOfTx returns the datastore storage item id of a transaction.
func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) {
id, ok := l.txIndex[txhash]
return id, ok
meta, ok := l.txIndex[txhash]
if !ok {
return 0, false
}
return meta.id, true
}
// storeidOfBlob returns the datastore storage item id of a blob.
@ -61,6 +70,15 @@ func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) {
return 0, false // Weird, don't choke
}
// sizeOfTx returns the RLP-encoded size of transaction
func (l *lookup) sizeOfTx(txhash common.Hash) (uint64, bool) {
meta, ok := l.txIndex[txhash]
if !ok {
return 0, false
}
return meta.size, true
}
// track inserts a new set of mappings from blob versioned hashes to transaction
// hashes; and from transaction hashes to datastore storage item ids.
func (l *lookup) track(tx *blobTxMeta) {
@ -71,8 +89,11 @@ func (l *lookup) track(tx *blobTxMeta) {
}
l.blobIndex[vhash][tx.hash] = struct{}{} // may be double mapped if a tx contains the same blob twice
}
// Map the transaction hash to the datastore id
l.txIndex[tx.hash] = tx.id
// Map the transaction hash to the datastore id and RLP-encoded transaction size
l.txIndex[tx.hash] = &txMetadata{
id: tx.id,
size: tx.size,
}
}
// untrack removes a set of mappings from blob versioned hashes to transaction

View file

@ -1035,6 +1035,19 @@ func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
return encoded
}
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
tx := pool.all.Get(hash)
if tx == nil {
return nil
}
return &txpool.TxMetadata{
Type: tx.Type(),
Size: tx.Size(),
}
}
// GetBlobs is not supported by the legacy transaction pool, it is just here to
// implement the txpool.SubPool interface.
func (pool *LegacyPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) {

View file

@ -86,6 +86,12 @@ type PendingFilter struct {
OnlyBlobTxs bool // Return only blob transactions (block blob-space filling)
}
// TxMetadata denotes the metadata of a transaction.
type TxMetadata struct {
Type uint8 // The type of the transaction
Size uint64 // The length of the 'rlp encoding' of a transaction
}
// SubPool represents a specialized transaction pool that lives on its own (e.g.
// blob pool). Since independent of how many specialized pools we have, they do
// need to be updated in lockstep and assemble into one coherent view for block
@ -127,6 +133,10 @@ type SubPool interface {
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
GetRLP(hash common.Hash) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
GetMetadata(hash common.Hash) *TxMetadata
// GetBlobs returns a number of blobs are proofs for the given versioned hashes.
// This is a utility method for the engine API, enabling consensus clients to
// retrieve blobs from the pools directly instead of the network.

View file

@ -354,6 +354,17 @@ func (p *TxPool) GetRLP(hash common.Hash) []byte {
return nil
}
// GetMetadata returns the transaction type and transaction size with the given
// hash.
func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata {
for _, subpool := range p.subpools {
if meta := subpool.GetMetadata(hash); meta != nil {
return meta
}
}
return nil
}
// GetBlobs returns a number of blobs are proofs for the given versioned hashes.
// This is a utility method for the engine API, enabling consensus clients to
// retrieve blobs from the pools directly instead of the network.

View file

@ -94,7 +94,7 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb
}
var bn uint64
if number == rpc.EarliestBlockNumber {
bn = b.eth.blockchain.HistoryPruningCutoff()
bn = b.HistoryPruningCutoff()
} else {
bn = uint64(number)
}
@ -152,10 +152,10 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe
}
bn := uint64(number) // the resolved number
if number == rpc.EarliestBlockNumber {
bn = b.eth.blockchain.HistoryPruningCutoff()
bn = b.HistoryPruningCutoff()
}
block := b.eth.blockchain.GetBlockByNumber(bn)
if block == nil && bn < b.eth.blockchain.HistoryPruningCutoff() {
if block == nil && bn < b.HistoryPruningCutoff() {
return nil, &ethconfig.PrunedHistoryError{}
}
return block, nil
@ -167,7 +167,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ
return nil, nil
}
block := b.eth.blockchain.GetBlock(hash, *number)
if block == nil && *number < b.eth.blockchain.HistoryPruningCutoff() {
if block == nil && *number < b.HistoryPruningCutoff() {
return nil, &ethconfig.PrunedHistoryError{}
}
return block, nil
@ -180,7 +180,7 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp
}
body := b.eth.blockchain.GetBody(hash)
if body == nil {
if uint64(number) < b.eth.blockchain.HistoryPruningCutoff() {
if uint64(number) < b.HistoryPruningCutoff() {
return nil, &ethconfig.PrunedHistoryError{}
}
return nil, errors.New("block body not found")
@ -202,7 +202,7 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
}
block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64())
if block == nil {
if header.Number.Uint64() < b.eth.blockchain.HistoryPruningCutoff() {
if header.Number.Uint64() < b.HistoryPruningCutoff() {
return nil, &ethconfig.PrunedHistoryError{}
}
return nil, errors.New("header found, but block body is missing")
@ -265,7 +265,8 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN
}
func (b *EthAPIBackend) HistoryPruningCutoff() uint64 {
return b.eth.blockchain.HistoryPruningCutoff()
bn, _ := b.eth.blockchain.HistoryPruningCutoff()
return bn
}
func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {

View file

@ -154,13 +154,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
// Validate history pruning configuration.
var historyPruningCutoff uint64
var (
cutoffNumber uint64
cutoffHash common.Hash
)
if config.HistoryMode == ethconfig.PostMergeHistory {
prunecfg, ok := ethconfig.HistoryPrunePoints[genesisHash]
if !ok {
return nil, fmt.Errorf("no history pruning point is defined for genesis %x", genesisHash)
}
historyPruningCutoff = prunecfg.BlockNumber
cutoffNumber = prunecfg.BlockNumber
cutoffHash = prunecfg.BlockHash
log.Info("Chain cutoff configured", "number", cutoffNumber, "hash", cutoffHash)
}
// Set networkID to chainID by default.
@ -204,16 +209,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
EnablePreimageRecording: config.EnablePreimageRecording,
}
cacheConfig = &core.CacheConfig{
TrieCleanLimit: config.TrieCleanCache,
TrieCleanNoPrefetch: config.NoPrefetch,
TrieDirtyLimit: config.TrieDirtyCache,
TrieDirtyDisabled: config.NoPruning,
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateHistory: config.StateHistory,
StateScheme: scheme,
HistoryPruningCutoff: historyPruningCutoff,
TrieCleanLimit: config.TrieCleanCache,
TrieCleanNoPrefetch: config.NoPrefetch,
TrieDirtyLimit: config.TrieDirtyCache,
TrieDirtyDisabled: config.NoPruning,
TrieTimeLimit: config.TrieTimeout,
SnapshotLimit: config.SnapshotCache,
Preimages: config.Preimages,
StateHistory: config.StateHistory,
StateScheme: scheme,
HistoryPruningCutoffNumber: cutoffNumber,
HistoryPruningCutoffHash: cutoffHash,
}
)
if config.VMTrace != "" {
@ -246,7 +252,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
HashScheme: scheme == rawdb.HashScheme,
}
chainView := eth.newChainView(eth.blockchain.CurrentBlock())
historyCutoff := eth.blockchain.HistoryPruningCutoff()
historyCutoff, _ := eth.blockchain.HistoryPruningCutoff()
var finalBlock uint64
if fb := eth.blockchain.CurrentFinalBlock(); fb != nil {
finalBlock = fb.Number.Uint64()
@ -443,7 +449,7 @@ func (s *Ethereum) updateFilterMapsHeads() {
if head == nil || newHead.Hash() != head.Hash() {
head = newHead
chainView := s.newChainView(head)
historyCutoff := s.blockchain.HistoryPruningCutoff()
historyCutoff, _ := s.blockchain.HistoryPruningCutoff()
var finalBlock uint64
if fb := s.blockchain.CurrentFinalBlock(); fb != nil {
finalBlock = fb.Number.Uint64()

View file

@ -273,8 +273,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) {
// fetchHeaders feeds skeleton headers to the downloader queue for scheduling
// until sync errors or is finished.
func (d *Downloader) fetchHeaders(from uint64) error {
var head *types.Header
_, tail, _, err := d.skeleton.Bounds()
head, tail, _, err := d.skeleton.Bounds()
if err != nil {
return err
}
@ -294,6 +293,27 @@ func (d *Downloader) fetchHeaders(from uint64) error {
fsHeaderContCheckTimer := time.NewTimer(fsHeaderContCheck)
defer fsHeaderContCheckTimer.Stop()
// Verify the header at configured chain cutoff, ensuring it's matched with
// the configured hash. Skip the check if the configured cutoff is even higher
// than the sync target, which is definitely not a common case.
if d.chainCutoffNumber != 0 && d.chainCutoffNumber >= from && d.chainCutoffNumber <= head.Number.Uint64() {
h := d.skeleton.Header(d.chainCutoffNumber)
if h == nil {
if d.chainCutoffNumber < tail.Number.Uint64() {
dist := tail.Number.Uint64() - d.chainCutoffNumber
if len(localHeaders) >= int(dist) {
h = localHeaders[dist-1]
}
}
}
if h == nil {
return fmt.Errorf("header at chain cutoff is not available, cutoff: %d", d.chainCutoffNumber)
}
if h.Hash() != d.chainCutoffHash {
return fmt.Errorf("header at chain cutoff mismatched, want: %v, got: %v", d.chainCutoffHash, h.Hash())
}
}
for {
// Some beacon headers might have appeared since the last cycle, make
// sure we're always syncing to all available ones

View file

@ -20,6 +20,7 @@ package downloader
import (
"errors"
"fmt"
"sort"
"sync"
"sync/atomic"
"time"
@ -120,6 +121,12 @@ type Downloader struct {
committed atomic.Bool
ancientLimit uint64 // The maximum block number which can be regarded as ancient data.
// The cutoff block number and hash before which chain segments (bodies
// and receipts) are skipped during synchronization. 0 means the entire
// chain segment is aimed for synchronization.
chainCutoffNumber uint64
chainCutoffHash common.Hash
// Channels
headerProcCh chan *headerTask // Channel to feed the header processor new tasks
@ -163,9 +170,6 @@ type BlockChain interface {
// CurrentHeader retrieves the head header from the local chain.
CurrentHeader() *types.Header
// InsertHeaderChain inserts a batch of headers into the local chain.
InsertHeaderChain([]*types.Header) (int, error)
// SetHead rewinds the local chain to a new head.
SetHead(uint64) error
@ -187,10 +191,17 @@ type BlockChain interface {
// SnapSyncCommitHead directly commits the head block to a certain entity.
SnapSyncCommitHead(common.Hash) error
// InsertHeadersBeforeCutoff inserts a batch of headers before the configured
// chain cutoff into the ancient store.
InsertHeadersBeforeCutoff([]*types.Header) (int, error)
// InsertChain inserts a batch of blocks into the local chain.
InsertChain(types.Blocks) (int, error)
// InsertReceiptChain inserts a batch of receipts into the local chain.
// InsertReceiptChain inserts a batch of blocks along with their receipts
// into the local chain. Blocks older than the specified `ancientLimit`
// are stored directly in the ancient store, while newer blocks are stored
// in the live key-value store.
InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error)
// Snapshots returns the blockchain snapshot tree to paused it during sync.
@ -199,22 +210,29 @@ type BlockChain interface {
// TrieDB retrieves the low level trie database used for interacting
// with trie nodes.
TrieDB() *triedb.Database
// HistoryPruningCutoff returns the configured history pruning point.
// Block bodies along with the receipts will be skipped for synchronization.
HistoryPruningCutoff() (uint64, common.Hash)
}
// New creates a new downloader to fetch hashes and blocks from remote peers.
func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader {
cutoffNumber, cutoffHash := chain.HistoryPruningCutoff()
dl := &Downloader{
stateDB: stateDb,
mux: mux,
queue: newQueue(blockCacheMaxItems, blockCacheInitialItems),
peers: newPeerSet(),
blockchain: chain,
dropPeer: dropPeer,
headerProcCh: make(chan *headerTask, 1),
quitCh: make(chan struct{}),
SnapSyncer: snap.NewSyncer(stateDb, chain.TrieDB().Scheme()),
stateSyncStart: make(chan *stateSync),
syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(),
stateDB: stateDb,
mux: mux,
queue: newQueue(blockCacheMaxItems, blockCacheInitialItems),
peers: newPeerSet(),
blockchain: chain,
chainCutoffNumber: cutoffNumber,
chainCutoffHash: cutoffHash,
dropPeer: dropPeer,
headerProcCh: make(chan *headerTask, 1),
quitCh: make(chan struct{}),
SnapSyncer: snap.NewSyncer(stateDb, chain.TrieDB().Scheme()),
stateSyncStart: make(chan *stateSync),
syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(),
}
// Create the post-merge skeleton syncer and start the process
dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success))
@ -503,6 +521,12 @@ func (d *Downloader) syncToHead() (err error) {
} else {
d.ancientLimit = 0
}
// Extend the ancient chain segment range if the ancient limit is even
// below the pre-configured chain cutoff.
if d.chainCutoffNumber != 0 && d.chainCutoffNumber > d.ancientLimit {
d.ancientLimit = d.chainCutoffNumber
log.Info("Extend the ancient range with configured cutoff", "cutoff", d.chainCutoffNumber)
}
frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here.
// If a part of blockchain data has already been written into active store,
@ -521,14 +545,23 @@ func (d *Downloader) syncToHead() (err error) {
log.Info("Truncated excess ancient chain segment", "oldhead", frozen-1, "newhead", origin)
}
}
// Skip ancient chain segments if Geth is running with a configured chain cutoff.
// These segments are not guaranteed to be available in the network.
chainOffset := origin + 1
if mode == ethconfig.SnapSync && d.chainCutoffNumber != 0 {
if chainOffset < d.chainCutoffNumber {
chainOffset = d.chainCutoffNumber
log.Info("Skip chain segment before cutoff", "origin", origin, "cutoff", d.chainCutoffNumber)
}
}
// Initiate the sync using a concurrent header and content retrieval algorithm
d.queue.Prepare(origin+1, mode)
d.queue.Prepare(chainOffset, mode)
// In beacon mode, headers are served by the skeleton syncer
fetchers := []func() error{
func() error { return d.fetchHeaders(origin + 1) }, // Headers are always retrieved
func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and snap sync
func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during snap sync
func() error { return d.fetchHeaders(origin + 1) }, // Headers are always retrieved
func() error { return d.fetchBodies(chainOffset) }, // Bodies are retrieved during normal and snap sync
func() error { return d.fetchReceipts(chainOffset) }, // Receipts are retrieved during snap sync
func() error { return d.processHeaders(origin + 1) },
}
if mode == ethconfig.SnapSync {
@ -666,7 +699,7 @@ func (d *Downloader) processHeaders(origin uint64) error {
return nil
}
// Otherwise split the chunk of headers into batches and process them
headers, hashes := task.headers, task.hashes
headers, hashes, scheduled := task.headers, task.hashes, false
for len(headers) > 0 {
// Terminate if something failed in between processing chunks
@ -683,17 +716,21 @@ func (d *Downloader) processHeaders(origin uint64) error {
chunkHeaders := headers[:limit]
chunkHashes := hashes[:limit]
// In case of header only syncing, validate the chunk immediately
if mode == ethconfig.SnapSync {
// Although the received headers might be all valid, a legacy
// PoW/PoA sync must not accept post-merge headers. Make sure
// that any transition is rejected at this point.
if len(chunkHeaders) > 0 {
if n, err := d.blockchain.InsertHeaderChain(chunkHeaders); err != nil {
log.Warn("Invalid header encountered", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err)
return fmt.Errorf("%w: %v", errInvalidChain, err)
}
// Split the headers around the chain cutoff
var cutoff int
if mode == ethconfig.SnapSync && d.chainCutoffNumber != 0 {
cutoff = sort.Search(len(chunkHeaders), func(i int) bool {
return chunkHeaders[i].Number.Uint64() >= d.chainCutoffNumber
})
}
// Insert the header chain into the ancient store (with block bodies and
// receipts set to nil) if they fall before the cutoff.
if mode == ethconfig.SnapSync && cutoff != 0 {
if n, err := d.blockchain.InsertHeadersBeforeCutoff(chunkHeaders[:cutoff]); err != nil {
log.Warn("Failed to insert ancient header chain", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err)
return fmt.Errorf("%w: %v", errInvalidChain, err)
}
log.Debug("Inserted headers before cutoff", "number", chunkHeaders[cutoff-1].Number, "hash", chunkHashes[cutoff-1])
}
// If we've reached the allowed number of pending headers, stall a bit
for d.queue.PendingBodies() >= maxQueuedHeaders || d.queue.PendingReceipts() >= maxQueuedHeaders {
@ -704,12 +741,21 @@ func (d *Downloader) processHeaders(origin uint64) error {
case <-timer.C:
}
}
// Otherwise insert the headers for content retrieval
inserts := d.queue.Schedule(chunkHeaders, chunkHashes, origin)
if len(inserts) != len(chunkHeaders) {
return fmt.Errorf("%w: stale headers", errBadPeer)
// Otherwise, schedule the headers for content retrieval (block bodies and
// potentially receipts in snap sync).
//
// Skip the bodies/receipts retrieval scheduling before the cutoff in snap
// sync if chain pruning is configured.
if mode == ethconfig.SnapSync && cutoff != 0 {
chunkHeaders = chunkHeaders[cutoff:]
chunkHashes = chunkHashes[cutoff:]
}
if len(chunkHeaders) > 0 {
scheduled = true
if d.queue.Schedule(chunkHeaders, chunkHashes, origin+uint64(cutoff)) != len(chunkHeaders) {
return fmt.Errorf("%w: stale headers", errBadPeer)
}
}
headers = headers[limit:]
hashes = hashes[limit:]
origin += uint64(limit)
@ -721,11 +767,13 @@ func (d *Downloader) processHeaders(origin uint64) error {
}
d.syncStatsLock.Unlock()
// Signal the content downloaders of the availability of new tasks
for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} {
select {
case ch <- true:
default:
// Signal the downloader of the availability of new tasks
if scheduled {
for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} {
select {
case ch <- true:
default:
}
}
}
}
@ -1085,10 +1133,20 @@ func (d *Downloader) reportSnapSyncProgress(force bool) {
header = d.blockchain.CurrentHeader()
block = d.blockchain.CurrentSnapBlock()
)
syncedBlocks := block.Number.Uint64() - d.syncStartBlock
if syncedBlocks == 0 {
// Prevent reporting if nothing has been synchronized yet
if block.Number.Uint64() <= d.syncStartBlock {
return
}
// Prevent reporting noise if the actual chain synchronization (headers
// and bodies) hasn't started yet. Inserting the ancient header chain is
// fast enough and would introduce significant bias if included in the count.
if d.chainCutoffNumber != 0 && block.Number.Uint64() <= d.chainCutoffNumber {
return
}
fetchedBlocks := block.Number.Uint64() - d.syncStartBlock
if d.chainCutoffNumber != 0 && d.chainCutoffNumber > d.syncStartBlock {
fetchedBlocks = block.Number.Uint64() - d.chainCutoffNumber
}
// Retrieve the current chain head and calculate the ETA
latest, _, _, err := d.skeleton.Bounds()
if err != nil {
@ -1103,7 +1161,7 @@ func (d *Downloader) reportSnapSyncProgress(force bool) {
}
var (
left = latest.Number.Uint64() - block.Number.Uint64()
eta = time.Since(d.syncStartTime) / time.Duration(syncedBlocks) * time.Duration(left)
eta = time.Since(d.syncStartTime) / time.Duration(fetchedBlocks) * time.Duration(left)
progress = fmt.Sprintf("%.2f%%", float64(block.Number.Uint64())*100/float64(latest.Number.Uint64()))
headers = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(header.Number.Uint64()), common.StorageSize(headerBytes).TerminalString())

View file

@ -25,7 +25,6 @@ import (
var (
headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil)
headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil)
headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil)
headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil)
bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil)

View file

@ -73,7 +73,7 @@ type fetchResult struct {
Withdrawals types.Withdrawals
}
func newFetchResult(header *types.Header, fastSync bool) *fetchResult {
func newFetchResult(header *types.Header, snapSync bool) *fetchResult {
item := &fetchResult{
Header: header,
}
@ -82,7 +82,7 @@ func newFetchResult(header *types.Header, fastSync bool) *fetchResult {
} else if header.WithdrawalsHash != nil {
item.Withdrawals = make(types.Withdrawals, 0)
}
if fastSync && !header.EmptyReceipts() {
if snapSync && !header.EmptyReceipts() {
item.pending.Store(item.pending.Load() | (1 << receiptType))
}
return item
@ -124,19 +124,8 @@ func (f *fetchResult) Done(kind uint) bool {
// queue represents hashes that are either need fetching or are being fetched
type queue struct {
mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching
// Headers are "special", they download in batches, supported by a skeleton chain
headerHead common.Hash // Hash of the last queued header to verify order
headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers
headerTaskQueue *prque.Prque[int64, uint64] // Priority queue of the skeleton indexes to fetch the filling headers for
headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable
headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations
headerResults []*types.Header // Result cache accumulating the completed headers
headerHashes []common.Hash // Result cache accumulating the completed header hashes
headerProced int // Number of headers already processed from the results
headerOffset uint64 // Number of the first header in the result cache
headerContCh chan bool // Channel to notify when header download finishes
mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching
headerHead common.Hash // Hash of the last queued header to verify order
// All data retrievals below are based on an already assembles header chain
blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers
@ -163,7 +152,6 @@ type queue struct {
func newQueue(blockCacheLimit int, thresholdInitialSize int) *queue {
lock := new(sync.RWMutex)
q := &queue{
headerContCh: make(chan bool, 1),
blockTaskQueue: prque.New[int64, *types.Header](nil),
blockWakeCh: make(chan bool, 1),
receiptTaskQueue: prque.New[int64, *types.Header](nil),
@ -182,9 +170,7 @@ func (q *queue) Reset(blockCacheLimit int, thresholdInitialSize int) {
q.closed = false
q.mode = ethconfig.FullSync
q.headerHead = common.Hash{}
q.headerPendPool = make(map[string]*fetchRequest)
q.blockTaskPool = make(map[common.Hash]*types.Header)
q.blockTaskQueue.Reset()
@ -207,14 +193,6 @@ func (q *queue) Close() {
q.lock.Unlock()
}
// PendingHeaders retrieves the number of header requests pending for retrieval.
func (q *queue) PendingHeaders() int {
q.lock.Lock()
defer q.lock.Unlock()
return q.headerTaskQueue.Size()
}
// PendingBodies retrieves the number of block body requests pending for retrieval.
func (q *queue) PendingBodies() int {
q.lock.Lock()
@ -260,54 +238,14 @@ func (q *queue) Idle() bool {
return (queued + pending) == 0
}
// ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill
// up an already retrieved header skeleton.
func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) {
q.lock.Lock()
defer q.lock.Unlock()
// No skeleton retrieval can be in progress, fail hard if so (huge implementation bug)
if q.headerResults != nil {
panic("skeleton assembly already in progress")
}
// Schedule all the header retrieval tasks for the skeleton assembly
q.headerTaskPool = make(map[uint64]*types.Header)
q.headerTaskQueue = prque.New[int64, uint64](nil)
q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains
q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch)
q.headerHashes = make([]common.Hash, len(skeleton)*MaxHeaderFetch)
q.headerProced = 0
q.headerOffset = from
q.headerContCh = make(chan bool, 1)
for i, header := range skeleton {
index := from + uint64(i*MaxHeaderFetch)
q.headerTaskPool[index] = header
q.headerTaskQueue.Push(index, -int64(index))
}
}
// RetrieveHeaders retrieves the header chain assemble based on the scheduled
// skeleton.
func (q *queue) RetrieveHeaders() ([]*types.Header, []common.Hash, int) {
q.lock.Lock()
defer q.lock.Unlock()
headers, hashes, proced := q.headerResults, q.headerHashes, q.headerProced
q.headerResults, q.headerHashes, q.headerProced = nil, nil, 0
return headers, hashes, proced
}
// Schedule adds a set of headers for the download queue for scheduling, returning
// the new headers encountered.
func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uint64) []*types.Header {
func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uint64) int {
q.lock.Lock()
defer q.lock.Unlock()
// Insert all the headers prioritised by the contained block number
inserts := make([]*types.Header, 0, len(headers))
var inserts int
for i, header := range headers {
// Make sure chain order is honoured and preserved throughout
hash := hashes[i]
@ -337,7 +275,7 @@ func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uin
q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64()))
}
}
inserts = append(inserts, header)
inserts++
q.headerHead = hash
from++
}
@ -390,7 +328,7 @@ func (q *queue) Results(block bool) []*fetchResult {
q.resultSize = common.StorageSize(blockCacheSizeWeight)*size +
(1-common.StorageSize(blockCacheSizeWeight))*q.resultSize
}
// Using the newly calibrated resultsize, figure out the new throttle limit
// Using the newly calibrated result size, figure out the new throttle limit
// on the result cache
throttleThreshold := uint64((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize)
throttleThreshold = q.resultCache.SetThrottleThreshold(throttleThreshold)
@ -428,46 +366,6 @@ func (q *queue) stats() []interface{} {
}
}
// ReserveHeaders reserves a set of headers for the given peer, skipping any
// previously failed batches.
func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest {
q.lock.Lock()
defer q.lock.Unlock()
// Short circuit if the peer's already downloading something (sanity check to
// not corrupt state)
if _, ok := q.headerPendPool[p.id]; ok {
return nil
}
// Retrieve a batch of hashes, skipping previously failed ones
send, skip := uint64(0), []uint64{}
for send == 0 && !q.headerTaskQueue.Empty() {
from, _ := q.headerTaskQueue.Pop()
if q.headerPeerMiss[p.id] != nil {
if _, ok := q.headerPeerMiss[p.id][from]; ok {
skip = append(skip, from)
continue
}
}
send = from
}
// Merge all the skipped batches back
for _, from := range skip {
q.headerTaskQueue.Push(from, -int64(from))
}
// Assemble and return the block download request
if send == 0 {
return nil
}
request := &fetchRequest{
Peer: p,
From: send,
Time: time.Now(),
}
q.headerPendPool[p.id] = request
return request
}
// ReserveBodies reserves a set of body fetches for the given peer, skipping any
// previously failed downloads. Beside the next batch of needed fetches, it also
// returns a flag whether empty blocks were queued requiring processing.
@ -594,10 +492,6 @@ func (q *queue) Revoke(peerID string) {
q.lock.Lock()
defer q.lock.Unlock()
if request, ok := q.headerPendPool[peerID]; ok {
q.headerTaskQueue.Push(request.From, -int64(request.From))
delete(q.headerPendPool, peerID)
}
if request, ok := q.blockPendPool[peerID]; ok {
for _, header := range request.Headers {
q.blockTaskQueue.Push(header, -int64(header.Number.Uint64()))
@ -612,16 +506,6 @@ func (q *queue) Revoke(peerID string) {
}
}
// ExpireHeaders cancels a request that timed out and moves the pending fetch
// task back into the queue for rescheduling.
func (q *queue) ExpireHeaders(peer string) int {
q.lock.Lock()
defer q.lock.Unlock()
headerTimeoutMeter.Mark(1)
return q.expire(peer, q.headerPendPool, q.headerTaskQueue)
}
// ExpireBodies checks for in flight block body requests that exceeded a timeout
// allowance, canceling them and returning the responsible peers for penalisation.
func (q *queue) ExpireBodies(peer string) int {
@ -670,116 +554,6 @@ func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue
return len(req.Headers)
}
// DeliverHeaders injects a header retrieval response into the header results
// cache. This method either accepts all headers it received, or none of them
// if they do not map correctly to the skeleton.
//
// If the headers are accepted, the method makes an attempt to deliver the set
// of ready headers to the processor to keep the pipeline full. However, it will
// not block to prevent stalling other pending deliveries.
func (q *queue) DeliverHeaders(id string, headers []*types.Header, hashes []common.Hash, headerProcCh chan *headerTask) (int, error) {
q.lock.Lock()
defer q.lock.Unlock()
var logger log.Logger
if len(id) < 16 {
// Tests use short IDs, don't choke on them
logger = log.New("peer", id)
} else {
logger = log.New("peer", id[:16])
}
// Short circuit if the data was never requested
request := q.headerPendPool[id]
if request == nil {
headerDropMeter.Mark(int64(len(headers)))
return 0, errNoFetchesPending
}
delete(q.headerPendPool, id)
headerReqTimer.UpdateSince(request.Time)
headerInMeter.Mark(int64(len(headers)))
// Ensure headers can be mapped onto the skeleton chain
target := q.headerTaskPool[request.From].Hash()
accepted := len(headers) == MaxHeaderFetch
if accepted {
if headers[0].Number.Uint64() != request.From {
logger.Trace("First header broke chain ordering", "number", headers[0].Number, "hash", hashes[0], "expected", request.From)
accepted = false
} else if hashes[len(headers)-1] != target {
logger.Trace("Last header broke skeleton structure ", "number", headers[len(headers)-1].Number, "hash", hashes[len(headers)-1], "expected", target)
accepted = false
}
}
if accepted {
parentHash := hashes[0]
for i, header := range headers[1:] {
hash := hashes[i+1]
if want := request.From + 1 + uint64(i); header.Number.Uint64() != want {
logger.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", want)
accepted = false
break
}
if parentHash != header.ParentHash {
logger.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash)
accepted = false
break
}
// Set-up parent hash for next round
parentHash = hash
}
}
// If the batch of headers wasn't accepted, mark as unavailable
if !accepted {
logger.Trace("Skeleton filling not accepted", "from", request.From)
headerDropMeter.Mark(int64(len(headers)))
miss := q.headerPeerMiss[id]
if miss == nil {
q.headerPeerMiss[id] = make(map[uint64]struct{})
miss = q.headerPeerMiss[id]
}
miss[request.From] = struct{}{}
q.headerTaskQueue.Push(request.From, -int64(request.From))
return 0, errors.New("delivery not accepted")
}
// Clean up a successful fetch and try to deliver any sub-results
copy(q.headerResults[request.From-q.headerOffset:], headers)
copy(q.headerHashes[request.From-q.headerOffset:], hashes)
delete(q.headerTaskPool, request.From)
ready := 0
for q.headerProced+ready < len(q.headerResults) && q.headerResults[q.headerProced+ready] != nil {
ready += MaxHeaderFetch
}
if ready > 0 {
// Headers are ready for delivery, gather them and push forward (non blocking)
processHeaders := make([]*types.Header, ready)
copy(processHeaders, q.headerResults[q.headerProced:q.headerProced+ready])
processHashes := make([]common.Hash, ready)
copy(processHashes, q.headerHashes[q.headerProced:q.headerProced+ready])
select {
case headerProcCh <- &headerTask{
headers: processHeaders,
hashes: processHashes,
}:
logger.Trace("Pre-scheduled new headers", "count", len(processHeaders), "from", processHeaders[0].Number)
q.headerProced += len(processHeaders)
default:
}
}
// Check for termination and return
if len(q.headerTaskPool) == 0 {
q.headerContCh <- false
}
return len(headers), nil
}
// DeliverBodies injects a block body retrieval response into the results queue.
// The method returns the number of blocks bodies accepted from the delivery and
// also wakes any threads waiting for data delivery.

View file

@ -76,7 +76,7 @@ func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 {
// throttled - if true, the store is at capacity, this particular header is not prio now
// item - the result to store data into
// err - any error that occurred
func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) {
func (r *resultStore) AddFetch(header *types.Header, snapSync bool) (stale, throttled bool, item *fetchResult, err error) {
r.lock.Lock()
defer r.lock.Unlock()
@ -86,7 +86,7 @@ func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, thro
return stale, throttled, item, err
}
if item == nil {
item = newFetchResult(header, fastSync)
item = newFetchResult(header, snapSync)
r.items[index] = item
}
return stale, throttled, item, err

View file

@ -275,7 +275,7 @@ func (s *skeleton) startup() {
for {
// If the sync cycle terminated or was terminated, propagate up when
// higher layers request termination. There's no fancy explicit error
// signalling as the sync loop should never terminate (TM).
// signaling as the sync loop should never terminate (TM).
newhead, err := s.sync(head)
switch {
case err == errSyncLinked:

View file

@ -71,6 +71,10 @@ type txPool interface {
// with given tx hash.
GetRLP(hash common.Hash) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
GetMetadata(hash common.Hash) *txpool.TxMetadata
// Add should add the given transactions to the pool.
Add(txs []*types.Transaction, sync bool) []error

View file

@ -93,6 +93,22 @@ func (p *testTxPool) GetRLP(hash common.Hash) []byte {
return nil
}
// GetMetadata returns the transaction type and transaction size with the given
// hash.
func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
if tx != nil {
return &txpool.TxMetadata{
Type: tx.Type(),
Size: tx.Size(),
}
}
return nil
}
// Add appends a batch of transactions to the pool, and notifies any
// listeners if the addition channel is non nil
func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {

View file

@ -116,10 +116,10 @@ func (p *Peer) announceTransactions() {
size common.StorageSize
)
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if tx := p.txpool.Get(queue[count]); tx != nil {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, tx.Type())
pendingSizes = append(pendingSizes, uint32(tx.Size()))
pendingTypes = append(pendingTypes, meta.Type)
pendingSizes = append(pendingSizes, uint32(meta.Size))
size += common.HashLength
}
}

View file

@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
@ -90,6 +91,10 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash.
GetRLP(hash common.Hash) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
GetMetadata(hash common.Hash) *txpool.TxMetadata
}
// MakeProtocols constructs the P2P protocol definitions for `eth`.

View file

@ -297,6 +297,34 @@ func testGetBlockHeaders(t *testing.T, protocol uint) {
backend.chain.GetBlockByNumber(0).Hash(),
},
},
// Check a corner case where skipping causes overflow with reverse=false
{
&GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1},
[]common.Hash{
backend.chain.GetBlockByNumber(1).Hash(),
},
},
// Check a corner case where skipping causes overflow with reverse=true
{
&GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: true, Skip: math.MaxUint64 - 1},
[]common.Hash{
backend.chain.GetBlockByNumber(1).Hash(),
},
},
// Check another corner case where skipping causes overflow with reverse=false
{
&GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: false, Skip: math.MaxUint64},
[]common.Hash{
backend.chain.GetBlockByNumber(1).Hash(),
},
},
// Check another corner case where skipping causes overflow with reverse=true
{
&GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: true, Skip: math.MaxUint64},
[]common.Hash{
backend.chain.GetBlockByNumber(1).Hash(),
},
},
// Check a corner case where skipping overflow loops back into the chain start
{
&GetBlockHeadersRequest{Origin: HashOrNumber{Hash: backend.chain.GetBlockByNumber(3).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1},

View file

@ -128,15 +128,22 @@ func serviceNonContiguousBlockHeaderQuery(chain *core.BlockChain, query *GetBloc
}
case query.Reverse:
// Number based traversal towards the genesis block
if query.Origin.Number >= query.Skip+1 {
query.Origin.Number -= query.Skip + 1
} else {
current := query.Origin.Number
ancestor := current - (query.Skip + 1)
if ancestor >= current { // check for underflow
unknown = true
} else {
query.Origin.Number = ancestor
}
case !query.Reverse:
// Number based traversal towards the leaf block
query.Origin.Number += query.Skip + 1
current := query.Origin.Number
next := current + query.Skip + 1
if next <= current { // check for overflow
unknown = true
} else {
query.Origin.Number = next
}
}
}
return headers

View file

@ -144,7 +144,7 @@ func (l panicLogger) Fatalf(format string, args ...interface{}) {
// New returns a wrapped pebble DB object. The namespace is the prefix that the
// metrics reporting should use for surfacing internal stats.
func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) {
func New(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (*Database, error) {
// Ensure we have some minimal caching and file guarantees
if cache < minCache {
cache = minCache
@ -185,7 +185,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
fn: file,
log: logger,
quitChan: make(chan chan error),
writeOptions: &pebble.WriteOptions{Sync: false},
writeOptions: &pebble.WriteOptions{Sync: !ephemeral},
}
opt := &pebble.Options{
// Pebble has a single combined cache area and the write

View file

@ -623,7 +623,10 @@ func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
panic("implement me")
}
func (b testBackend) HistoryPruningCutoff() uint64 { return b.chain.HistoryPruningCutoff() }
func (b testBackend) HistoryPruningCutoff() uint64 {
bn, _ := b.chain.HistoryPruningCutoff()
return bn
}
func TestEstimateGas(t *testing.T) {
t.Parallel()

View file

@ -15,7 +15,7 @@
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// This file contains the code snippets from the developer's guide embedded into
// Go tests. This ensures that any code published in out guides will not break
// Go tests. This ensures that any code published in our guides will not break
// accidentally via some code update. If some API changes nonetheless that needs
// modifying this file, please port any modification over into the developer's
// guide wiki pages too!

View file

@ -36,6 +36,11 @@ type openOptions struct {
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
ReadOnly bool
// Ephemeral means that filesystem sync operations should be avoided:
// data integrity in the face of a crash is not important. This option
// should typically be used in tests.
Ephemeral bool
}
// openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also
@ -78,7 +83,7 @@ func openKeyValueDatabase(o openOptions) (ethdb.Database, error) {
}
if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble {
log.Info("Using pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
}
if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb {
log.Info("Using leveldb as the backing database")
@ -86,7 +91,7 @@ func openKeyValueDatabase(o openOptions) (ethdb.Database, error) {
}
// No pre-existing database, no user-requested one either. Default to Pebble.
log.Info("Defaulting to pebble as the backing database")
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
}
// newLevelDBDatabase creates a persistent key-value database without a freezer
@ -102,8 +107,8 @@ func newLevelDBDatabase(file string, cache int, handles int, namespace string, r
// newPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly)
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (ethdb.Database, error) {
db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
if err != nil {
return nil, err
}

View file

@ -39,7 +39,7 @@ const talkHandlerLaunchTimeout = 400 * time.Millisecond
// Note that talk handlers are expected to come up with a response very quickly, within at
// most 200ms or so. If the handler takes longer than that, the remote end may time out
// and wont receive the response.
type TalkRequestHandler func(enode.ID, *net.UDPAddr, []byte) []byte
type TalkRequestHandler func(*enode.Node, *net.UDPAddr, []byte) []byte
type talkSystem struct {
transport *UDPv5
@ -72,13 +72,19 @@ func (t *talkSystem) register(protocol string, handler TalkRequestHandler) {
// handleRequest handles a talk request.
func (t *talkSystem) handleRequest(id enode.ID, addr netip.AddrPort, req *v5wire.TalkRequest) {
n := t.transport.codec.SessionNode(id, addr.String())
if n == nil {
// The node must be contained in the session here, since we wouldn't have
// received the request otherwise.
panic("missing node in session")
}
t.mutex.Lock()
handler, ok := t.handlers[req.Protocol]
t.mutex.Unlock()
if !ok {
resp := &v5wire.TalkResponse{ReqID: req.ReqID}
t.transport.sendResponse(id, addr, resp)
t.transport.sendResponse(n.ID(), addr, resp)
return
}
@ -90,9 +96,9 @@ func (t *talkSystem) handleRequest(id enode.ID, addr netip.AddrPort, req *v5wire
go func() {
defer func() { t.slots <- struct{}{} }()
udpAddr := &net.UDPAddr{IP: addr.Addr().AsSlice(), Port: int(addr.Port())}
respMessage := handler(id, udpAddr, req.Message)
respMessage := handler(n, udpAddr, req.Message)
resp := &v5wire.TalkResponse{ReqID: req.ReqID, Message: respMessage}
t.transport.sendFromAnotherThread(id, addr, resp)
t.transport.sendFromAnotherThread(n.ID(), addr, resp)
}()
case <-timeout.C:
// Couldn't get it in time, drop the request.

View file

@ -64,6 +64,9 @@ type codecV5 interface {
// CurrentChallenge returns the most recent WHOAREYOU challenge that was encoded to given node.
// This will return a non-nil value if there is an active handshake attempt with the node, and nil otherwise.
CurrentChallenge(id enode.ID, addr string) *v5wire.Whoareyou
// SessionNode returns a node that has completed the handshake.
SessionNode(id enode.ID, addr string) *enode.Node
}
// UDPv5 is the implementation of protocol version 5.

View file

@ -181,29 +181,35 @@ func TestUDPv5_handshakeRepeatChallenge(t *testing.T) {
nonce1 := v5wire.Nonce{1}
nonce2 := v5wire.Nonce{2}
nonce3 := v5wire.Nonce{3}
check := func(p *v5wire.Whoareyou, wantNonce v5wire.Nonce) {
var firstAuthTag *v5wire.Nonce
check := func(p *v5wire.Whoareyou, authTag, wantNonce v5wire.Nonce) {
t.Helper()
if p.Nonce != wantNonce {
t.Error("wrong nonce in WHOAREYOU:", p.Nonce, wantNonce)
t.Error("wrong nonce in WHOAREYOU:", p.Nonce, "want:", wantNonce)
}
if firstAuthTag == nil {
firstAuthTag = &authTag
} else if authTag != *firstAuthTag {
t.Error("wrong auth tag in WHOAREYOU header:", authTag, "want:", *firstAuthTag)
}
}
// Unknown packet from unknown node.
test.packetIn(&v5wire.Unknown{Nonce: nonce1})
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) {
check(p, nonce1)
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) {
check(p, authTag, nonce1)
})
// Second unknown packet. Here we expect the response to reference the
// first unknown packet.
test.packetIn(&v5wire.Unknown{Nonce: nonce2})
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) {
check(p, nonce1)
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) {
check(p, authTag, nonce1)
})
// Third unknown packet. This should still return the first nonce.
test.packetIn(&v5wire.Unknown{Nonce: nonce3})
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) {
check(p, nonce1)
test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) {
check(p, authTag, nonce1)
})
}
@ -486,7 +492,7 @@ func TestUDPv5_talkHandling(t *testing.T) {
defer test.close()
var recvMessage []byte
test.udp.RegisterTalkHandler("test", func(id enode.ID, addr *net.UDPAddr, message []byte) []byte {
test.udp.RegisterTalkHandler("test", func(n *enode.Node, addr *net.UDPAddr, message []byte) []byte {
recvMessage = message
return []byte("test response")
})
@ -766,20 +772,30 @@ type testCodecFrame struct {
}
func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) {
// To match the behavior of v5wire.Codec, we return the cached encoding of
// WHOAREYOU challenges.
if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.Encoded) > 0 {
return wp.Encoded, wp.Nonce, nil
}
c.ctr++
var authTag v5wire.Nonce
binary.BigEndian.PutUint64(authTag[:], c.ctr)
penc, _ := rlp.EncodeToBytes(p)
frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc})
if err != nil {
return frame, authTag, err
}
// Store recently sent challenges.
if w, ok := p.(*v5wire.Whoareyou); ok {
// Store recently sent Whoareyou challenges.
w.Nonce = authTag
w.Encoded = frame
if c.sentChallenges == nil {
c.sentChallenges = make(map[enode.ID]*v5wire.Whoareyou)
}
c.sentChallenges[toID] = w
}
penc, _ := rlp.EncodeToBytes(p)
frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc})
return frame, authTag, err
}
@ -795,6 +811,10 @@ func (c *testCodec) Decode(input []byte, addr string) (enode.ID, *enode.Node, v5
return frame.NodeID, nil, p, nil
}
func (c *testCodec) SessionNode(id enode.ID, addr string) *enode.Node {
return c.test.nodesByID[id].Node()
}
func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Packet, err error) {
if err = rlp.DecodeBytes(input, &frame); err != nil {
return frame, nil, fmt.Errorf("invalid frame: %v", err)

View file

@ -189,6 +189,11 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar
)
switch {
case packet.Kind() == WhoareyouPacket:
// just send the WHOAREYOU packet raw again, rather than the re-encoded challenge data
w := packet.(*Whoareyou)
if len(w.Encoded) > 0 {
return w.Encoded, w.Nonce, nil
}
head, err = c.encodeWhoareyou(id, packet.(*Whoareyou))
case challenge != nil:
// We have an unanswered challenge, send handshake.
@ -218,15 +223,22 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar
// Store sent WHOAREYOU challenges.
if challenge, ok := packet.(*Whoareyou); ok {
challenge.ChallengeData = bytesCopy(&c.buf)
enc, err := c.EncodeRaw(id, head, msgData)
if err != nil {
return nil, Nonce{}, err
}
challenge.Encoded = bytes.Clone(enc)
c.sc.storeSentHandshake(id, addr, challenge)
} else if msgData == nil {
return enc, head.Nonce, err
}
if msgData == nil {
headerData := c.buf.Bytes()
msgData, err = c.encryptMessage(session, packet, &head, headerData)
if err != nil {
return nil, Nonce{}, err
}
}
enc, err := c.EncodeRaw(id, head, msgData)
return enc, head.Nonce, err
}
@ -347,7 +359,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who
}
// TODO: this should happen when the first authenticated message is received
c.sc.storeNewSession(toID, addr, session)
c.sc.storeNewSession(toID, addr, session, challenge.Node)
// Encode the auth header.
var (
@ -522,7 +534,7 @@ func (c *Codec) decodeHandshakeMessage(fromAddr string, head *Header, headerData
}
// Handshake OK, drop the challenge and store the new session keys.
c.sc.storeNewSession(auth.h.SrcID, fromAddr, session)
c.sc.storeNewSession(auth.h.SrcID, fromAddr, session, node)
c.sc.deleteHandshake(auth.h.SrcID, fromAddr)
return node, msg, nil
}
@ -644,6 +656,10 @@ func (c *Codec) decryptMessage(input, nonce, headerData, readKey []byte) (Packet
return DecodeMessage(msgdata[0], msgdata[1:])
}
func (c *Codec) SessionNode(id enode.ID, addr string) *enode.Node {
return c.sc.readNode(id, addr)
}
// checkValid performs some basic validity checks on the header.
// The packetLen here is the length remaining after the static header.
func (h *StaticHeader) checkValid(packetLen int, protocolID [6]byte) error {

View file

@ -166,7 +166,7 @@ func TestHandshake_rekey(t *testing.T) {
readKey: []byte("BBBBBBBBBBBBBBBB"),
writeKey: []byte("AAAAAAAAAAAAAAAA"),
}
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session)
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n())
// A -> B FINDNODE (encrypted with zero keys)
findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{})
@ -209,8 +209,8 @@ func TestHandshake_rekey2(t *testing.T) {
readKey: []byte("CCCCCCCCCCCCCCCC"),
writeKey: []byte("DDDDDDDDDDDDDDDD"),
}
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA)
net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB)
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA, net.nodeB.n())
net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB, net.nodeA.n())
// A -> B FINDNODE encrypted with initKeysA
findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{Distances: []uint{3}})
@ -362,8 +362,8 @@ func TestTestVectorsV5(t *testing.T) {
ENRSeq: 2,
},
prep: func(net *handshakeTest) {
net.nodeA.c.sc.storeNewSession(idB, addr, session)
net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped())
net.nodeA.c.sc.storeNewSession(idB, addr, session, net.nodeB.n())
net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped(), net.nodeA.n())
},
},
{
@ -499,8 +499,8 @@ func BenchmarkV5_DecodePing(b *testing.B) {
readKey: []byte{233, 203, 93, 195, 86, 47, 177, 186, 227, 43, 2, 141, 244, 230, 120, 17},
writeKey: []byte{79, 145, 252, 171, 167, 216, 252, 161, 208, 190, 176, 106, 214, 39, 178, 134},
}
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session)
net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped())
net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n())
net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped(), net.nodeA.n())
addrB := net.nodeA.addr()
ping := &Ping{ReqID: []byte("reqid"), ENRSeq: 5}
enc, _, err := net.nodeA.c.Encode(net.nodeB.id(), addrB, ping, nil)

View file

@ -73,6 +73,9 @@ type (
Node *enode.Node
sent mclock.AbsTime // for handshake GC.
// Encoded is packet raw data for sending out, but should not be include in the RLP encoding.
Encoded []byte `rlp:"-"`
}
// PING is sent during liveness checks.

View file

@ -54,11 +54,12 @@ type session struct {
writeKey []byte
readKey []byte
nonceCounter uint32
node *enode.Node
}
// keysFlipped returns a copy of s with the read and write keys flipped.
func (s *session) keysFlipped() *session {
return &session{s.readKey, s.writeKey, s.nonceCounter}
return &session{s.readKey, s.writeKey, s.nonceCounter, s.node}
}
func NewSessionCache(maxItems int, clock mclock.Clock) *SessionCache {
@ -103,8 +104,19 @@ func (sc *SessionCache) readKey(id enode.ID, addr string) []byte {
return nil
}
func (sc *SessionCache) readNode(id enode.ID, addr string) *enode.Node {
if s := sc.session(id, addr); s != nil {
return s.node
}
return nil
}
// storeNewSession stores new encryption keys in the cache.
func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session) {
func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session, n *enode.Node) {
if n == nil {
panic("nil node in storeNewSession")
}
s.node = n
sc.sessions.Add(sessionID{id, addr}, s)
}

View file

@ -36,8 +36,9 @@ import (
// transition, typically corresponding to a block execution. It can also represent
// the combined trie node set from several aggregated state transitions.
type nodeSet struct {
size uint64 // aggregated size of the trie node
nodes map[common.Hash]map[string]*trienode.Node // node set, mapped by owner and path
size uint64 // aggregated size of the trie node
accountNodes map[string]*trienode.Node // account trie nodes, mapped by path
storageNodes map[common.Hash]map[string]*trienode.Node // storage trie nodes, mapped by owner and path
}
// newNodeSet constructs the set with the provided dirty trie nodes.
@ -46,7 +47,17 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet {
if nodes == nil {
nodes = make(map[common.Hash]map[string]*trienode.Node)
}
s := &nodeSet{nodes: nodes}
s := &nodeSet{
accountNodes: make(map[string]*trienode.Node),
storageNodes: make(map[common.Hash]map[string]*trienode.Node),
}
for owner, subset := range nodes {
if owner == (common.Hash{}) {
s.accountNodes = subset
} else {
s.storageNodes[owner] = subset
}
}
s.computeSize()
return s
}
@ -54,13 +65,12 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet {
// computeSize calculates the database size of the held trie nodes.
func (s *nodeSet) computeSize() {
var size uint64
for owner, subset := range s.nodes {
var prefix int
if owner != (common.Hash{}) {
prefix = common.HashLength // owner (32 bytes) for storage trie nodes
}
for path, n := range s.accountNodes {
size += uint64(len(n.Blob) + len(path))
}
for _, subset := range s.storageNodes {
for path, n := range subset {
size += uint64(prefix + len(n.Blob) + len(path))
size += uint64(common.HashLength + len(n.Blob) + len(path))
}
}
s.size = size
@ -79,15 +89,18 @@ func (s *nodeSet) updateSize(delta int64) {
// node retrieves the trie node with node path and its trie identifier.
func (s *nodeSet) node(owner common.Hash, path []byte) (*trienode.Node, bool) {
subset, ok := s.nodes[owner]
// Account trie node
if owner == (common.Hash{}) {
n, ok := s.accountNodes[string(path)]
return n, ok
}
// Storage trie node
subset, ok := s.storageNodes[owner]
if !ok {
return nil, false
}
n, ok := subset[string(path)]
if !ok {
return nil, false
}
return n, true
return n, ok
}
// merge integrates the provided dirty nodes into the set. The provided nodeset
@ -97,15 +110,24 @@ func (s *nodeSet) merge(set *nodeSet) {
delta int64 // size difference resulting from node merging
overwrite counter // counter of nodes being overwritten
)
for owner, subset := range set.nodes {
var prefix int
if owner != (common.Hash{}) {
prefix = common.HashLength
// Merge account nodes
for path, n := range set.accountNodes {
if orig, exist := s.accountNodes[path]; !exist {
delta += int64(len(n.Blob) + len(path))
} else {
delta += int64(len(n.Blob) - len(orig.Blob))
overwrite.add(len(orig.Blob) + len(path))
}
current, exist := s.nodes[owner]
s.accountNodes[path] = n
}
// Merge storage nodes
for owner, subset := range set.storageNodes {
current, exist := s.storageNodes[owner]
if !exist {
for path, n := range subset {
delta += int64(prefix + len(n.Blob) + len(path))
delta += int64(common.HashLength + len(n.Blob) + len(path))
}
// Perform a shallow copy of the map for the subset instead of claiming it
// directly from the provided nodeset to avoid potential concurrent map
@ -113,19 +135,19 @@ func (s *nodeSet) merge(set *nodeSet) {
// accessible even after merging. Therefore, ownership of the nodes map
// should still belong to the original layer, and any modifications to it
// should be prevented.
s.nodes[owner] = maps.Clone(subset)
s.storageNodes[owner] = maps.Clone(subset)
continue
}
for path, n := range subset {
if orig, exist := current[path]; !exist {
delta += int64(prefix + len(n.Blob) + len(path))
delta += int64(common.HashLength + len(n.Blob) + len(path))
} else {
delta += int64(len(n.Blob) - len(orig.Blob))
overwrite.add(prefix + len(orig.Blob) + len(path))
overwrite.add(common.HashLength + len(orig.Blob) + len(path))
}
current[path] = n
}
s.nodes[owner] = current
s.storageNodes[owner] = current
}
overwrite.report(gcTrieNodeMeter, gcTrieNodeBytesMeter)
s.updateSize(delta)
@ -136,34 +158,38 @@ func (s *nodeSet) merge(set *nodeSet) {
func (s *nodeSet) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) {
var delta int64
for owner, subset := range nodes {
current, ok := s.nodes[owner]
if !ok {
panic(fmt.Sprintf("non-existent subset (%x)", owner))
}
for path, n := range subset {
orig, ok := current[path]
if !ok {
// There is a special case in merkle tree that one child is removed
// from a fullNode which only has two children, and then a new child
// with different position is immediately inserted into the fullNode.
// In this case, the clean child of the fullNode will also be marked
// as dirty because of node collapse and expansion. In case of database
// rollback, don't panic if this "clean" node occurs which is not
// present in buffer.
var blob []byte
if owner == (common.Hash{}) {
blob = rawdb.ReadAccountTrieNode(db, []byte(path))
} else {
blob = rawdb.ReadStorageTrieNode(db, owner, []byte(path))
if owner == (common.Hash{}) {
// Account trie nodes
for path, n := range subset {
orig, ok := s.accountNodes[path]
if !ok {
blob := rawdb.ReadAccountTrieNode(db, []byte(path))
if bytes.Equal(blob, n.Blob) {
continue
}
panic(fmt.Sprintf("non-existent account node (%v) blob: %v", path, crypto.Keccak256Hash(n.Blob).Hex()))
}
// Ignore the clean node in the case described above.
if bytes.Equal(blob, n.Blob) {
continue
}
panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex()))
s.accountNodes[path] = n
delta += int64(len(n.Blob)) - int64(len(orig.Blob))
}
} else {
// Storage trie nodes
current, ok := s.storageNodes[owner]
if !ok {
panic(fmt.Sprintf("non-existent subset (%x)", owner))
}
for path, n := range subset {
orig, ok := current[path]
if !ok {
blob := rawdb.ReadStorageTrieNode(db, owner, []byte(path))
if bytes.Equal(blob, n.Blob) {
continue
}
panic(fmt.Sprintf("non-existent storage node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex()))
}
current[path] = n
delta += int64(len(n.Blob)) - int64(len(orig.Blob))
}
current[path] = n
delta += int64(len(n.Blob)) - int64(len(orig.Blob))
}
}
s.updateSize(delta)
@ -184,8 +210,21 @@ type journalNodes struct {
// encode serializes the content of trie nodes into the provided writer.
func (s *nodeSet) encode(w io.Writer) error {
nodes := make([]journalNodes, 0, len(s.nodes))
for owner, subset := range s.nodes {
nodes := make([]journalNodes, 0, len(s.storageNodes)+1)
// Encode account nodes
if len(s.accountNodes) > 0 {
entry := journalNodes{Owner: common.Hash{}}
for path, node := range s.accountNodes {
entry.Nodes = append(entry.Nodes, journalNode{
Path: []byte(path),
Blob: node.Blob,
})
}
nodes = append(nodes, entry)
}
// Encode storage nodes
for owner, subset := range s.storageNodes {
entry := journalNodes{Owner: owner}
for path, node := range subset {
entry.Nodes = append(entry.Nodes, journalNode{
@ -204,43 +243,61 @@ func (s *nodeSet) decode(r *rlp.Stream) error {
if err := r.Decode(&encoded); err != nil {
return fmt.Errorf("load nodes: %v", err)
}
nodes := make(map[common.Hash]map[string]*trienode.Node)
s.accountNodes = make(map[string]*trienode.Node)
s.storageNodes = make(map[common.Hash]map[string]*trienode.Node)
for _, entry := range encoded {
subset := make(map[string]*trienode.Node)
for _, n := range entry.Nodes {
if len(n.Blob) > 0 {
subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
} else {
subset[string(n.Path)] = trienode.NewDeleted()
if entry.Owner == (common.Hash{}) {
// Account nodes
for _, n := range entry.Nodes {
if len(n.Blob) > 0 {
s.accountNodes[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
} else {
s.accountNodes[string(n.Path)] = trienode.NewDeleted()
}
}
} else {
// Storage nodes
subset := make(map[string]*trienode.Node)
for _, n := range entry.Nodes {
if len(n.Blob) > 0 {
subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
} else {
subset[string(n.Path)] = trienode.NewDeleted()
}
}
s.storageNodes[entry.Owner] = subset
}
nodes[entry.Owner] = subset
}
s.nodes = nodes
s.computeSize()
return nil
}
// write flushes nodes into the provided database batch as a whole.
func (s *nodeSet) write(batch ethdb.Batch, clean *fastcache.Cache) int {
return writeNodes(batch, s.nodes, clean)
nodes := make(map[common.Hash]map[string]*trienode.Node)
if len(s.accountNodes) > 0 {
nodes[common.Hash{}] = s.accountNodes
}
for owner, subset := range s.storageNodes {
nodes[owner] = subset
}
return writeNodes(batch, nodes, clean)
}
// reset clears all cached trie node data.
func (s *nodeSet) reset() {
s.nodes = make(map[common.Hash]map[string]*trienode.Node)
s.accountNodes = make(map[string]*trienode.Node)
s.storageNodes = make(map[common.Hash]map[string]*trienode.Node)
s.size = 0
}
// dbsize returns the approximate size of db write.
func (s *nodeSet) dbsize() int {
var m int
for owner, nodes := range s.nodes {
if owner == (common.Hash{}) {
m += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix
} else {
m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix
}
m += len(s.accountNodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix
for _, nodes := range s.storageNodes {
m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix
}
return m + int(s.size)
}