From fd5078c779f2aab50ea759bd3dcd3c5d4f2556f8 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Thu, 13 Jun 2024 23:52:46 -0700 Subject: [PATCH 01/12] trie/triedb: add Reader to backend interface (#29988) --- triedb/database.go | 12 +++++------- triedb/hashdb/database.go | 3 ++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/triedb/database.go b/triedb/database.go index 10f77982f3..ef757e7f5b 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -74,6 +74,10 @@ type backend interface { // Close closes the trie database backend and releases all held resources. Close() error + + // Reader returns a reader for accessing all trie nodes with provided state + // root. An error will be returned if the requested state is not available. + Reader(root common.Hash) (database.Reader, error) } // Database is the wrapper of the underlying backend which is shared by different @@ -123,13 +127,7 @@ func NewDatabase(diskdb ethdb.Database, config *Config) *Database { // Reader returns a reader for accessing all trie nodes with provided state root. // An error will be returned if the requested state is not available. func (db *Database) Reader(blockRoot common.Hash) (database.Reader, error) { - switch b := db.backend.(type) { - case *hashdb.Database: - return b.Reader(blockRoot) - case *pathdb.Database: - return b.Reader(blockRoot) - } - return nil, errors.New("unknown backend") + return db.backend.Reader(blockRoot) } // Update performs a state transition by committing dirty nodes contained in the diff --git a/triedb/hashdb/database.go b/triedb/hashdb/database.go index ebb5d72057..bb0deca9a7 100644 --- a/triedb/hashdb/database.go +++ b/triedb/hashdb/database.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" + "github.com/ethereum/go-ethereum/triedb/database" ) var ( @@ -625,7 +626,7 @@ func (db *Database) Close() error { // Reader retrieves a node reader belonging to the given state root. // An error will be returned if the requested state is not available. -func (db *Database) Reader(root common.Hash) (*reader, error) { +func (db *Database) Reader(root common.Hash) (database.Reader, error) { if _, err := db.node(root); err != nil { return nil, fmt.Errorf("state %#x is not available, %v", root, err) } From 34b46a2f756da71595ac84eb7f25441f2a5b6ebb Mon Sep 17 00:00:00 2001 From: maskpp Date: Mon, 17 Jun 2024 15:42:39 +0800 Subject: [PATCH 02/12] core/state/snapshot: add a missing lock (#30001) * upgrade lock usage * revert unnecessary change --- core/state/snapshot/snapshot.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 89a4c16c20..7c76765417 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -829,6 +829,8 @@ func (t *Tree) disklayer() *diskLayer { case *diskLayer: return layer case *diffLayer: + layer.lock.RLock() + defer layer.lock.RUnlock() return layer.origin default: panic(fmt.Sprintf("%T: undefined layer", snap)) @@ -860,7 +862,7 @@ func (t *Tree) generating() (bool, error) { return layer.genMarker != nil, nil } -// DiskRoot is a external helper function to return the disk layer root. +// DiskRoot is an external helper function to return the disk layer root. func (t *Tree) DiskRoot() common.Hash { t.lock.Lock() defer t.lock.Unlock() From a58e4f06747b287af6d5befaddaae66c4494f431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 17 Jun 2024 11:15:27 +0300 Subject: [PATCH 03/12] go.mod: update Pebble to sort out a deleted upstream dependency (#30010) --- go.mod | 11 ++++++----- go.sum | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 9947df31f3..b578ec07b5 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.2.0 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v1.1.0 + github.com/cockroachdb/pebble v1.1.1 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c github.com/crate-crypto/go-kzg-4844 v1.0.0 @@ -61,7 +61,7 @@ require ( github.com/rs/cors v1.7.0 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/supranational/blst v0.3.11 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 @@ -95,7 +95,8 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect @@ -105,7 +106,7 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61 // indirect - github.com/getsentry/sentry-go v0.18.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -116,7 +117,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/klauspost/compress v1.15.15 // indirect + github.com/klauspost/compress v1.16.0 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index 5e4b38e9a6..168637b0ad 100644 --- a/go.sum +++ b/go.sum @@ -116,12 +116,14 @@ github.com/cloudflare/cloudflare-go v0.79.0/go.mod h1:gkHQf9xEubaQPEuerBuoinR9P8 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= -github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= -github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= -github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= +github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -187,8 +189,8 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILD github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= -github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -347,8 +349,8 @@ github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4 github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -494,8 +496,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= From b78d2352ef55b727ed57d1abe612af09c67be29a Mon Sep 17 00:00:00 2001 From: Zoro <40222601+BabyHalimao@users.noreply.github.com> Date: Mon, 17 Jun 2024 17:03:27 +0800 Subject: [PATCH 04/12] log: fix some functions comments (#29907) updates some docstrings --------- Co-authored-by: rjl493456442 --- log/handler_glog.go | 12 ++++++++++-- log/logger.go | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/log/handler_glog.go b/log/handler_glog.go index 625a036403..739f8c5b42 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -139,11 +139,15 @@ func (h *GlogHandler) Vmodule(ruleset string) error { return nil } +// Enabled implements slog.Handler, reporting whether the handler handles records +// at the given level. func (h *GlogHandler) Enabled(ctx context.Context, lvl slog.Level) bool { // fast-track skipping logging if override not enabled and the provided verbosity is above configured return h.override.Load() || slog.Level(h.level.Load()) <= lvl } +// WithAttrs implements slog.Handler, returning a new Handler whose attributes +// consist of both the receiver's attributes and the arguments. func (h *GlogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { h.lock.RLock() siteCache := maps.Clone(h.siteCache) @@ -164,12 +168,16 @@ func (h *GlogHandler) WithAttrs(attrs []slog.Attr) slog.Handler { return &res } +// WithGroup implements slog.Handler, returning a new Handler with the given +// group appended to the receiver's existing groups. +// +// Note, this function is not implemented. func (h *GlogHandler) WithGroup(name string) slog.Handler { panic("not implemented") } -// Log implements Handler.Log, filtering a log record through the global, local -// and backtrace filters, finally emitting it if either allow it through. +// Handle implements slog.Handler, filtering a log record through the global, +// local and backtrace filters, finally emitting it if either allow it through. func (h *GlogHandler) Handle(_ context.Context, r slog.Record) error { // If the global log level allows, fast track logging if slog.Level(h.level.Load()) <= r.Level { diff --git a/log/logger.go b/log/logger.go index 8b03b68fc8..016856c834 100644 --- a/log/logger.go +++ b/log/logger.go @@ -35,7 +35,7 @@ const ( LvlDebug = LevelDebug ) -// convert from old Geth verbosity level constants +// FromLegacyLevel converts from old Geth verbosity level constants // to levels defined by slog func FromLegacyLevel(lvl int) slog.Level { switch lvl { @@ -107,7 +107,7 @@ type Logger interface { // With returns a new Logger that has this logger's attributes plus the given attributes With(ctx ...interface{}) Logger - // With returns a new Logger that has this logger's attributes plus the given attributes. Identical to 'With'. + // New returns a new Logger that has this logger's attributes plus the given attributes. Identical to 'With'. New(ctx ...interface{}) Logger // Log logs a message at the specified level with context key/value pairs @@ -156,7 +156,7 @@ func (l *logger) Handler() slog.Handler { return l.inner.Handler() } -// Write logs a message at the specified level: +// Write logs a message at the specified level. func (l *logger) Write(level slog.Level, msg string, attrs ...any) { if !l.inner.Enabled(context.Background(), level) { return From 115d154392dc47eedbdfe9e598f2f4d9ae421976 Mon Sep 17 00:00:00 2001 From: maskpp Date: Mon, 17 Jun 2024 17:42:41 +0800 Subject: [PATCH 05/12] trie, triedb/pathdb: prealloc capacity for map and slice (#29986) --- trie/secure_trie.go | 2 +- trie/trienode/node.go | 2 +- triedb/pathdb/history.go | 14 ++++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index fb39a80609..cfa7f0bddb 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -224,7 +224,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - preimages := make(map[common.Hash][]byte) + preimages := make(map[common.Hash][]byte, len(t.secKeyCache)) for hk, key := range t.secKeyCache { preimages[common.BytesToHash([]byte(hk))] = key } diff --git a/trie/trienode/node.go b/trie/trienode/node.go index aa8a0f6d99..fa7bebe4f9 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -186,7 +186,7 @@ func (set *MergedNodeSet) Merge(other *NodeSet) error { // Flatten returns a two-dimensional map for internal nodes. func (set *MergedNodeSet) Flatten() map[common.Hash]map[string]*Node { - nodes := make(map[common.Hash]map[string]*Node) + nodes := make(map[common.Hash]map[string]*Node, len(set.Sets)) for owner, set := range set.Sets { nodes[owner] = set.Nodes } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 3663cbbdb9..60ce3dc2bc 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -384,10 +384,11 @@ func (r *decoder) readAccount(pos int) (accountIndex, []byte, error) { func (r *decoder) readStorage(accIndex accountIndex) ([]common.Hash, map[common.Hash][]byte, error) { var ( last common.Hash - list []common.Hash - storage = make(map[common.Hash][]byte) + count = int(accIndex.storageSlots) + list = make([]common.Hash, 0, count) + storage = make(map[common.Hash][]byte, count) ) - for j := 0; j < int(accIndex.storageSlots); j++ { + for j := 0; j < count; j++ { var ( index slotIndex start = (accIndex.storageOffset + uint32(j)) * uint32(slotIndexSize) @@ -430,9 +431,10 @@ func (r *decoder) readStorage(accIndex accountIndex) ([]common.Hash, map[common. // decode deserializes the account and storage data from the provided byte stream. func (h *history) decode(accountData, storageData, accountIndexes, storageIndexes []byte) error { var ( - accounts = make(map[common.Address][]byte) + count = len(accountIndexes) / accountIndexSize + accounts = make(map[common.Address][]byte, count) storages = make(map[common.Address]map[common.Hash][]byte) - accountList []common.Address + accountList = make([]common.Address, 0, count) storageList = make(map[common.Address][]common.Hash) r = &decoder{ @@ -445,7 +447,7 @@ func (h *history) decode(accountData, storageData, accountIndexes, storageIndexe if err := r.verify(); err != nil { return err } - for i := 0; i < len(accountIndexes)/accountIndexSize; i++ { + for i := 0; i < count; i++ { // Resolve account first accIndex, accData, err := r.readAccount(i) if err != nil { From c736b04d9b3bec8d9281146490b05075a91e7eea Mon Sep 17 00:00:00 2001 From: maskpp Date: Mon, 17 Jun 2024 23:09:29 +0800 Subject: [PATCH 06/12] triedb/pathdb: use maps.Clone and maps.Keys (#29985) --- triedb/pathdb/history.go | 11 +++-------- triedb/pathdb/nodebuffer.go | 5 ++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 60ce3dc2bc..d77f7aa04d 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triestate" + "golang.org/x/exp/maps" ) // State history records the state changes involved in executing a block. The @@ -244,19 +245,13 @@ type history struct { // newHistory constructs the state history object with provided state change set. func newHistory(root common.Hash, parent common.Hash, block uint64, states *triestate.Set) *history { var ( - accountList []common.Address + accountList = maps.Keys(states.Accounts) storageList = make(map[common.Address][]common.Hash) ) - for addr := range states.Accounts { - accountList = append(accountList, addr) - } slices.SortFunc(accountList, common.Address.Cmp) for addr, slots := range states.Storages { - slist := make([]common.Hash, 0, len(slots)) - for slotHash := range slots { - slist = append(slist, slotHash) - } + slist := maps.Keys(slots) slices.SortFunc(slist, common.Hash.Cmp) storageList[addr] = slist } diff --git a/triedb/pathdb/nodebuffer.go b/triedb/pathdb/nodebuffer.go index ff09484100..d3492602c8 100644 --- a/triedb/pathdb/nodebuffer.go +++ b/triedb/pathdb/nodebuffer.go @@ -19,6 +19,7 @@ package pathdb import ( "bytes" "fmt" + "maps" "time" "github.com/VictoriaMetrics/fastcache" @@ -90,12 +91,10 @@ func (b *nodebuffer) commit(nodes map[common.Hash]map[string]*trienode.Node) *no // The nodes belong to original diff layer are still accessible even // after merging, thus the ownership of nodes map should still belong // to original layer and any mutation on it should be prevented. - current = make(map[string]*trienode.Node, len(subset)) for path, n := range subset { - current[path] = n delta += int64(len(n.Blob) + len(path)) } - b.nodes[owner] = current + b.nodes[owner] = maps.Clone(subset) continue } for path, n := range subset { From d8664490da47bfff1854869826268c486e6487d7 Mon Sep 17 00:00:00 2001 From: Dean Eigenmann <7621705+decanus@users.noreply.github.com> Date: Mon, 17 Jun 2024 21:53:00 +0200 Subject: [PATCH 07/12] common/math: fix out of bounds access in json unmarshalling (#30014) Co-authored-by: Martin Holst Swende --- common/math/big.go | 2 +- common/math/integer.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/math/big.go b/common/math/big.go index 721b297c9c..d9748d01a3 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -54,7 +54,7 @@ func NewHexOrDecimal256(x int64) *HexOrDecimal256 { // It is similar to UnmarshalText, but allows parsing real decimals too, not just // quoted decimal strings. func (i *HexOrDecimal256) UnmarshalJSON(input []byte) error { - if len(input) > 0 && input[0] == '"' { + if len(input) > 1 && input[0] == '"' { input = input[1 : len(input)-1] } return i.UnmarshalText(input) diff --git a/common/math/integer.go b/common/math/integer.go index 080fba8fea..82de96f927 100644 --- a/common/math/integer.go +++ b/common/math/integer.go @@ -46,7 +46,7 @@ type HexOrDecimal64 uint64 // It is similar to UnmarshalText, but allows parsing real decimals too, not just // quoted decimal strings. func (i *HexOrDecimal64) UnmarshalJSON(input []byte) error { - if len(input) > 0 && input[0] == '"' { + if len(input) > 1 && input[0] == '"' { input = input[1 : len(input)-1] } return i.UnmarshalText(input) From 7cf6a63687da2d9300fec495ff4bc978c4b70b65 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 18 Jun 2024 04:52:49 +0200 Subject: [PATCH 08/12] core/state/snapshot: acquire the lock on Release (#30011) * core/state/snapshot: acquire the lock on release * core/state/snapshot: only acquire read-lock when iterating --- core/state/snapshot/snapshot.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 7c76765417..752f4359fb 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -666,6 +666,9 @@ func diffToDisk(bottom *diffLayer) *diskLayer { // Release releases resources func (t *Tree) Release() { + t.lock.RLock() + defer t.lock.RUnlock() + if dl := t.disklayer(); dl != nil { dl.Release() } @@ -850,8 +853,8 @@ func (t *Tree) diskRoot() common.Hash { // generating is an internal helper function which reports whether the snapshot // is still under the construction. func (t *Tree) generating() (bool, error) { - t.lock.Lock() - defer t.lock.Unlock() + t.lock.RLock() + defer t.lock.RUnlock() layer := t.disklayer() if layer == nil { @@ -864,8 +867,8 @@ func (t *Tree) generating() (bool, error) { // DiskRoot is an external helper function to return the disk layer root. func (t *Tree) DiskRoot() common.Hash { - t.lock.Lock() - defer t.lock.Unlock() + t.lock.RLock() + defer t.lock.RUnlock() return t.diskRoot() } From 67a862db9de372cd9c53fb3d7f44252d6b535c28 Mon Sep 17 00:00:00 2001 From: Ha DANG Date: Wed, 19 Jun 2024 13:47:17 +0700 Subject: [PATCH 09/12] cmd/geth, ethdb/pebble: improve database statistic (#29948) * cmd/geth, ethdb/pebble: polish method naming and code comment * implement db stat for pebble * cmd, core, ethdb, internal, trie: remove db property selector * cmd, core, ethdb: fix function description --------- Co-authored-by: prpeh Co-authored-by: Gary Rong --- cmd/geth/chaincmd.go | 4 +-- cmd/geth/dbcmd.go | 20 ++++++-------- core/rawdb/table.go | 6 ++--- ethdb/database.go | 4 +-- ethdb/leveldb/leveldb.go | 54 +++++++++++++++++++++++++++++++------ ethdb/memorydb/memorydb.go | 6 ++--- ethdb/pebble/pebble.go | 6 ++--- ethdb/remotedb/remotedb.go | 4 +-- internal/ethapi/api.go | 4 +-- internal/web3ext/web3ext.go | 1 - trie/trie_test.go | 2 +- 11 files changed, 71 insertions(+), 40 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2965b99d94..9450c09e7e 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -336,7 +336,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Import done in %v.\n\n", time.Since(start)) // Output pre-compaction stats mostly to see the import trashing - showLeveldbStats(db) + showDBStats(db) // Print the memory statistics used by the importing mem := new(runtime.MemStats) @@ -359,7 +359,7 @@ func importChain(ctx *cli.Context) error { } fmt.Printf("Compaction done in %v.\n\n", time.Since(start)) - showLeveldbStats(db) + showDBStats(db) return importErr } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 742eadd5f3..4b683569a4 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -407,17 +407,13 @@ func checkStateContent(ctx *cli.Context) error { return nil } -func showLeveldbStats(db ethdb.KeyValueStater) { - if stats, err := db.Stat("leveldb.stats"); err != nil { +func showDBStats(db ethdb.KeyValueStater) { + stats, err := db.Stat() + if err != nil { log.Warn("Failed to read database stats", "error", err) - } else { - fmt.Println(stats) - } - if ioStats, err := db.Stat("leveldb.iostats"); err != nil { - log.Warn("Failed to read database iostats", "error", err) - } else { - fmt.Println(ioStats) + return } + fmt.Println(stats) } func dbStats(ctx *cli.Context) error { @@ -427,7 +423,7 @@ func dbStats(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() - showLeveldbStats(db) + showDBStats(db) return nil } @@ -439,7 +435,7 @@ func dbCompact(ctx *cli.Context) error { defer db.Close() log.Info("Stats before compaction") - showLeveldbStats(db) + showDBStats(db) log.Info("Triggering compaction") if err := db.Compact(nil, nil); err != nil { @@ -447,7 +443,7 @@ func dbCompact(ctx *cli.Context) error { return err } log.Info("Stats after compaction") - showLeveldbStats(db) + showDBStats(db) return nil } diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 19e4ed5b5c..90849fe9d9 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -147,9 +147,9 @@ func (t *table) NewIterator(prefix []byte, start []byte) ethdb.Iterator { } } -// Stat returns a particular internal stat of the database. -func (t *table) Stat(property string) (string, error) { - return t.db.Stat(property) +// Stat returns the statistic data of the database. +func (t *table) Stat() (string, error) { + return t.db.Stat() } // Compact flattens the underlying data store for the given key range. In essence, diff --git a/ethdb/database.go b/ethdb/database.go index 3ec1f70e3b..9ef5942a13 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -39,8 +39,8 @@ type KeyValueWriter interface { // KeyValueStater wraps the Stat method of a backing data store. type KeyValueStater interface { - // Stat returns a particular internal stat of the database. - Stat(property string) (string, error) + // Stat returns the statistic data of the database. + Stat() (string, error) } // Compacter wraps the Compact method of a backing data store. diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 64f51cf217..92838ad7ab 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -22,7 +22,6 @@ package leveldb import ( "fmt" - "strings" "sync" "time" @@ -244,14 +243,53 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { return &snapshot{db: snap}, nil } -// Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - if property == "" { - property = "leveldb.stats" - } else if !strings.HasPrefix(property, "leveldb.") { - property = "leveldb." + property +// Stat returns the statistic data of the database. +func (db *Database) Stat() (string, error) { + var stats leveldb.DBStats + if err := db.db.Stats(&stats); err != nil { + return "", err } - return db.db.GetProperty(property) + var ( + message string + totalRead int64 + totalWrite int64 + totalSize int64 + totalTables int + totalDuration time.Duration + ) + if len(stats.LevelSizes) > 0 { + message += " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" + + "-------+------------+---------------+---------------+---------------+---------------\n" + for level, size := range stats.LevelSizes { + read := stats.LevelRead[level] + write := stats.LevelWrite[level] + duration := stats.LevelDurations[level] + tables := stats.LevelTablesCounts[level] + + if tables == 0 && duration == 0 { + continue + } + totalTables += tables + totalSize += size + totalRead += read + totalWrite += write + totalDuration += duration + message += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", + level, tables, float64(size)/1048576.0, duration.Seconds(), + float64(read)/1048576.0, float64(write)/1048576.0) + } + message += "-------+------------+---------------+---------------+---------------+---------------\n" + message += fmt.Sprintf(" Total | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", + totalTables, float64(totalSize)/1048576.0, totalDuration.Seconds(), + float64(totalRead)/1048576.0, float64(totalWrite)/1048576.0) + message += "-------+------------+---------------+---------------+---------------+---------------\n\n" + } + message += fmt.Sprintf("Read(MB):%.5f Write(MB):%.5f\n", float64(stats.IORead)/1048576.0, float64(stats.IOWrite)/1048576.0) + message += fmt.Sprintf("BlockCache(MB):%.5f FileCache:%d\n", float64(stats.BlockCacheSize)/1048576.0, stats.OpenedTablesCount) + message += fmt.Sprintf("MemoryCompaction:%d Level0Compaction:%d NonLevel0Compaction:%d SeekCompaction:%d\n", stats.MemComp, stats.Level0Comp, stats.NonLevel0Comp, stats.SeekComp) + message += fmt.Sprintf("WriteDelayCount:%d WriteDelayDuration:%s Paused:%t\n", stats.WriteDelayCount, common.PrettyDuration(stats.WriteDelayDuration), stats.WritePaused) + message += fmt.Sprintf("Snapshots:%d Iterators:%d\n", stats.AliveSnapshots, stats.AliveIterators) + return message, nil } // Compact flattens the underlying data store for the given key range. In essence, diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index d1233acb21..9b0872f89a 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -182,9 +182,9 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { return newSnapshot(db), nil } -// Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - return "", errors.New("unknown property") +// Stat returns the statistic data of the database. +func (db *Database) Stat() (string, error) { + return "", nil } // Compact is not supported on a memory database, but there's no need either as diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 0fac07c960..130d6617b5 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -416,10 +416,8 @@ func upperBound(prefix []byte) (limit []byte) { } // Stat returns the internal metrics of Pebble in a text format. It's a developer -// method to read everything there is to read independent of Pebble version. -// -// The property is unused in Pebble as there's only one thing to retrieve. -func (d *Database) Stat(property string) (string, error) { +// method to read everything there is to read, independent of Pebble version. +func (d *Database) Stat() (string, error) { return d.db.Metrics().String(), nil } diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index c1c803caf2..bfecde4a32 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -126,8 +126,8 @@ func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { panic("not supported") } -func (db *Database) Stat(property string) (string, error) { - panic("not supported") +func (db *Database) Stat() (string, error) { + return "", nil } func (db *Database) AncientDatadir() (string, error) { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1c209fb5a4..0ecedf1130 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -2108,8 +2108,8 @@ func (api *DebugAPI) PrintBlock(ctx context.Context, number uint64) (string, err } // ChaindbProperty returns leveldb properties of the key-value database. -func (api *DebugAPI) ChaindbProperty(property string) (string, error) { - return api.b.ChainDb().Stat(property) +func (api *DebugAPI) ChaindbProperty() (string, error) { + return api.b.ChainDb().Stat() } // ChaindbCompact flattens the entire key-value database into a single level, diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 1da7d737dd..4a1a37d722 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -263,7 +263,6 @@ web3._extend({ new web3._extend.Method({ name: 'chaindbProperty', call: 'debug_chaindbProperty', - params: 1, outputFormatter: console.log }), new web3._extend.Method({ diff --git a/trie/trie_test.go b/trie/trie_test.go index f31fd393f5..5f706a28be 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -820,7 +820,7 @@ func (s *spongeDb) Delete(key []byte) error { panic("implement func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} } func (s *spongeDb) NewBatchWithSize(size int) ethdb.Batch { return &spongeBatch{s} } func (s *spongeDb) NewSnapshot() (ethdb.Snapshot, error) { panic("implement me") } -func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") } +func (s *spongeDb) Stat() (string, error) { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } func (s *spongeDb) Put(key []byte, value []byte) error { From 0e3a0a693ce99cca1f928c913108237d4be8c233 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Wed, 19 Jun 2024 00:58:22 -0700 Subject: [PATCH 10/12] trie: don't reset tracer at the end of Commit (#30024) * trie: don't reset tracer at the end of Commit * Update trie.go --------- Co-authored-by: rjl493456442 --- trie/trie.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/trie/trie.go b/trie/trie.go index 935f81fc7d..e1a9201108 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -53,7 +53,6 @@ type Trie struct { reader *trieReader // tracer is the tool to track the trie changes. - // It will be reset after each commit operation. tracer *tracer } @@ -609,7 +608,6 @@ func (t *Trie) Hash() common.Hash { // Once the trie is committed, it's not usable anymore. A new trie must // be created with new root and updated trie database for following usage func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { - defer t.tracer.reset() defer func() { t.committed = true }() From c11aac249ddca275a12a1da0e66d1f4c6c21f4f6 Mon Sep 17 00:00:00 2001 From: Halimao <1065621723@qq.com> Date: Wed, 19 Jun 2024 17:06:52 +0800 Subject: [PATCH 11/12] common: using `ParseUint` instead of `ParseInt` (#30020) Since Decimal is defined as unsiged `uint64`, we should use `strconv.ParseUint` instead of `strconv.ParseInt` during unmarshalling. --------- Co-authored-by: Martin Holst Swende --- common/types.go | 2 +- common/types_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/common/types.go b/common/types.go index b914787d13..fdb25f1b34 100644 --- a/common/types.go +++ b/common/types.go @@ -468,7 +468,7 @@ func (d *Decimal) UnmarshalJSON(input []byte) error { if !isString(input) { return &json.UnmarshalTypeError{Value: "non-string", Type: reflect.TypeOf(uint64(0))} } - if i, err := strconv.ParseInt(string(input[1:len(input)-1]), 10, 64); err == nil { + if i, err := strconv.ParseUint(string(input[1:len(input)-1]), 10, 64); err == nil { *d = Decimal(i) return nil } else { diff --git a/common/types_test.go b/common/types_test.go index cec689ea39..11247b117b 100644 --- a/common/types_test.go +++ b/common/types_test.go @@ -21,6 +21,7 @@ import ( "database/sql/driver" "encoding/json" "fmt" + "math" "math/big" "reflect" "strings" @@ -595,3 +596,29 @@ func BenchmarkPrettyDuration(b *testing.B) { } b.Logf("Post %s", a) } + +func TestDecimalUnmarshalJSON(t *testing.T) { + // These should error + for _, tc := range []string{``, `"`, `""`, `"-1"`} { + if err := new(Decimal).UnmarshalJSON([]byte(tc)); err == nil { + t.Errorf("input %s should cause error", tc) + } + } + // These should succeed + for _, tc := range []struct { + input string + want uint64 + }{ + {`"0"`, 0}, + {`"9223372036854775807"`, math.MaxInt64}, + {`"18446744073709551615"`, math.MaxUint64}, + } { + have := new(Decimal) + if err := have.UnmarshalJSON([]byte(tc.input)); err != nil { + t.Errorf("input %q triggered error: %v", tc.input, err) + } + if uint64(*have) != tc.want { + t.Errorf("input %q, have %d want %d", tc.input, *have, tc.want) + } + } +} From 27008408a57c77e55d3630adb50c72b0a36abb32 Mon Sep 17 00:00:00 2001 From: psogv0308 Date: Wed, 19 Jun 2024 21:46:57 +0900 Subject: [PATCH 12/12] core/txpool/blobpool: change rw-lock to r-lock (#29989) --- core/txpool/blobpool/blobpool.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index f1c2c10fc9..69423731ee 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1598,8 +1598,8 @@ func (p *BlobPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool // Nonce returns the next nonce of an account, with all transactions executable // by the pool already applied on top. func (p *BlobPool) Nonce(addr common.Address) uint64 { - p.lock.Lock() - defer p.lock.Unlock() + p.lock.RLock() + defer p.lock.RUnlock() if txs, ok := p.index[addr]; ok { return txs[len(txs)-1].nonce + 1 @@ -1610,8 +1610,8 @@ func (p *BlobPool) Nonce(addr common.Address) uint64 { // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *BlobPool) Stats() (int, int) { - p.lock.Lock() - defer p.lock.Unlock() + p.lock.RLock() + defer p.lock.RUnlock() var pending int for _, txs := range p.index {