forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-25 16:06:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ethdb/pebble, cmd, node: add pebble v2 support (#34009)
Some checks are pending
/ Linux Build (push) Waiting to run
Details
/ Linux Build (arm) (push) Waiting to run
Details
/ Keeper Build (push) Waiting to run
Details
/ Windows Build (push) Waiting to run
Details
/ Docker Image (push) Waiting to run
Details

Browse source 
This PR adds the support of Pebble v2, details as below:

- Pebble V2 will be used if database is empty
- Pebble V1 will be used if database is not empty and the format is old
- Upgrade command (geth db pebble-upgrade) is provided to upgrade the
format to v2 offline
This commit is contained in:
rjl493456442 2026-06-22 18:50:49 +08:00 committed by GitHub
parent c80681ca73
commit 7f3afa1b71
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1260 additions and 58 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cmd/geth/dbcmd.go
View file

@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/pebble"
"github.com/ethereum/go-ethereum/internal/tablewriter"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
@ -102,6 +103,7 @@ Remove blockchain and state databases`,
dbMetadataCmd,
dbCheckStateContentCmd,
dbInspectHistoryCmd,
dbPebbleUpgradeCmd,
},
}
dbInspectCmd = &cli.Command{
@ -242,6 +244,17 @@ WARNING: This is a low-level operation which may cause database corruption!`,
}, utils.NetworkFlags, utils.DatabaseFlags),
Description: "This command queries the history of the account or storage slot within the specified block range",
}
dbPebbleUpgradeCmd = &cli.Command{
Action: dbPebbleUpgrade,
Name: "pebble-upgrade",
Usage: "Upgrade a legacy pebble v1 database to pebble v2 format",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: `This command upgrades a legacy Pebble v1 database so
that it becomes compatible with Pebble v2. The upgrade process converts the
database format to the oldest format supported by Pebble v2. It's not the
one-way operation, instead, the database can still be opened by older versions
of geth that use the pebble v1 library.`,
}
)
func removeDB(ctx *cli.Context) error {
@ -543,6 +556,21 @@ func dbCompact(ctx *cli.Context) error {
return nil
}
func dbPebbleUpgrade(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := stack.ResolvePath("chaindata")
dbType := rawdb.PreexistingDatabase(path)
if dbType == "" {
return fmt.Errorf("no database found at %s", path)
}
if dbType != rawdb.DBPebble {
return fmt.Errorf("database at %s is %s, not pebble", path, dbType)
}
return pebble.Upgrade(path)
}
// dbGet shows the value of a given database key
func dbGet(ctx *cli.Context) error {
if ctx.NArg() != 1 {

28
cmd/keeper/go.sum
View file

@ -1,7 +1,11 @@
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU=
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI=
github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8=
github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y=
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk=
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMGMsNAoA/+4G0=
@ -14,6 +18,8 @@ github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3M
github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU=
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
@ -22,8 +28,12 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
github.com/cockroachdb/pebble/v2 v2.1.4 h1:j9wPgMDbkErFdAKYFGhsoCcvzcjR+6zrJ4jhKtJ6bOk=
github.com/cockroachdb/pebble/v2 v2.1.4/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8=
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI=
github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI=
@ -72,8 +82,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ
github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA=
github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA=
github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -87,6 +97,8 @@ github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzW
github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw=
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
@ -95,14 +107,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/prysmaticlabs/gohashtree v0.0.4-beta h1:H/EbCuXPeTV3lpKeXGPpEV9gsUpkqOOVnWapUyeWro4=
github.com/prysmaticlabs/gohashtree v0.0.4-beta/go.mod h1:BFdtALS+Ffhg3lGQIHv9HDWuHS8cTvHZzrHWxwOtGOs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=

37
core/rawdb/database.go
View file

@ -352,17 +352,40 @@ const (
// PreexistingDatabase checks the given data directory whether a database is already
// instantiated at that location, and if so, returns the type of database (or the
// empty string).
//
// The database flavors are told apart by their on-disk file layout:
//
// CURRENT marker.manifest.* OPTIONS*
// leveldb x
// pebble v1 x x
// pebble v2 x x
func PreexistingDatabase(path string) string {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
var (
hasCurrent = fileExists(filepath.Join(path, "CURRENT"))
hasMarker = anyFileMatches(filepath.Join(path, "marker.manifest.*"))
hasOptions = anyFileMatches(filepath.Join(path, "OPTIONS*"))
)
switch {
case hasMarker, hasCurrent && hasOptions:
return DBPebble
case hasCurrent:
return DBLeveldb
default:
return "" // No pre-existing db
}
if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil {
if err != nil {
panic(err) // only possible if the pattern is malformed
}
return DBPebble
}
func fileExists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
func anyFileMatches(pattern string) bool {
matches, err := filepath.Glob(pattern)
if err != nil {
panic(err) // only possible if the pattern is malformed
}
return DBLeveldb
return len(matches) > 0
}
type counter uint64

68
ethdb/pebble/pebble.go
View file

@ -18,6 +18,7 @@
package pebble
import (
"context"
"errors"
"fmt"
"runtime"
@ -26,8 +27,8 @@ import (
"sync/atomic"
"time"
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/bloom"
"github.com/cockroachdb/pebble/v2"
"github.com/cockroachdb/pebble/v2/bloom"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
@ -52,7 +53,7 @@ const (
degradationWarnInterval = time.Minute
)
// Database is a persistent key-value store based on the pebble storage engine.
// Database is a persistent key-value store based on the pebble v2 storage engine.
// Apart from basic data storage functionality it also supports batch writes and
// iterating over the keyspace in binary-alphabetical order.
type Database struct {
@ -77,8 +78,8 @@ type Database struct {
zombieMemTablesGauge *metrics.Gauge // Gauge for tracking the number of zombie memory tables
blockCacheHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in the block cache
blockCacheMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in the block cache
tableCacheHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in the table cache
tableCacheMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in the table cache
tableCacheHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in the file cache
tableCacheMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in the file cache
filterHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in bloom filter
filterMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in bloom filter
estimatedCompDebtGauge *metrics.Gauge // Gauge for tracking the number of bytes that need to be compacted
@ -186,7 +187,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
handles = minHandles
}
logger := log.New("database", file)
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles)
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles, "version", "v2")
// The max memtable size is limited by the uint32 offsets stored in
// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
@ -232,6 +233,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
// of course). Geth is expected to handle recovery from an unclean shutdown.
writeOptions: pebble.NoSync,
}
numCPU := runtime.NumCPU()
opt := &pebble.Options{
// Pebble has a single combined cache area and the write
// buffers are taken from this too. Assign all available
@ -256,20 +258,30 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
// The default compaction concurrency(1 thread),
// Here use all available CPUs for faster compaction.
MaxConcurrentCompactions: runtime.NumCPU,
CompactionConcurrencyRange: func() (int, int) { return 1, numCPU },
// Per-level options. Options for at least one level must be specified. The
// options for the last level are used for all subsequent levels.
Levels: []pebble.LevelOptions{
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 4 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 8 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 32 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 64 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
Levels: [7]pebble.LevelOptions{
{FilterPolicy: bloom.FilterPolicy(10)},
{FilterPolicy: bloom.FilterPolicy(10)},
{FilterPolicy: bloom.FilterPolicy(10)},
{FilterPolicy: bloom.FilterPolicy(10)},
{FilterPolicy: bloom.FilterPolicy(10)},
{FilterPolicy: bloom.FilterPolicy(10)},
// Pebble doesn't use the Bloom filter at level6 for read efficiency.
{TargetFileSize: 128 * 1024 * 1024},
{},
},
// Per-level target file sizes (replaces LevelOptions.TargetFileSize in v2).
TargetFileSizes: [7]int64{
2 * 1024 * 1024,
4 * 1024 * 1024,
8 * 1024 * 1024,
16 * 1024 * 1024,
32 * 1024 * 1024,
64 * 1024 * 1024,
128 * 1024 * 1024,
},
ReadOnly: readonly,
EventListener: &pebble.EventListener{
@ -299,6 +311,14 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
// the compaction debt as around 10GB. By reducing it to 2, the compaction
// debt will be less than 1GB, but with more frequent compactions scheduled.
L0CompactionThreshold: 2,
// FormatFlushableIngest is the minimum FormatMajorVersion supported by
// pebble v2. The more advanced version can be enabled later.
//
// This version is supported by both v1 and v2. It serves as the natural
// bridge point: a v1 database can be ratcheted up to FormatFlushableIngest
// using pebble v1, and then pebble v2 can open it since that's its minimum.
FormatMajorVersion: formatMinV2,
}
// Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130
// for more details.
@ -309,7 +329,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) (
// - there is one more overlapping sub-level0;
// - there is an additional 256 MB of compaction debt;
//
// The maximum concurrency is still capped by MaxConcurrentCompactions, but with
// The maximum concurrency is still capped by CompactionConcurrencyRange, but with
// these settings compactions can scale up more readily.
opt.Experimental.L0CompactionConcurrency = 1
opt.Experimental.CompactionDebtConcurrency = 1 << 28 // 256MB
@ -506,7 +526,7 @@ func (d *Database) Compact(start []byte, limit []byte) error {
if limit == nil {
limit = ethdb.MaximumKey
}
return d.db.Compact(start, limit, true) // Parallelization is preferred
return d.db.Compact(context.Background(), start, limit, true) // Parallelization is preferred
}
// Path returns the path to the database directory.
@ -565,10 +585,10 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
compTimes[i%2] = compTime
for _, levelMetrics := range stats.Levels {
nWrite += int64(levelMetrics.BytesCompacted)
nWrite += int64(levelMetrics.BytesFlushed)
compWrite += int64(levelMetrics.BytesCompacted)
compRead += int64(levelMetrics.BytesRead)
nWrite += int64(levelMetrics.TableBytesCompacted)
nWrite += int64(levelMetrics.TableBytesFlushed)
compWrite += int64(levelMetrics.TableBytesCompacted)
compRead += int64(levelMetrics.TableBytesRead)
}
nWrite += int64(stats.WAL.BytesWritten)
@ -607,8 +627,8 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
d.liveMemTablesGauge.Update(stats.MemTable.Count)
d.zombieMemTablesGauge.Update(stats.MemTable.ZombieCount)
d.estimatedCompDebtGauge.Update(int64(stats.Compact.EstimatedDebt))
d.tableCacheHitGauge.Update(stats.TableCache.Hits)
d.tableCacheMissGauge.Update(stats.TableCache.Misses)
d.tableCacheHitGauge.Update(stats.FileCache.Hits)
d.tableCacheMissGauge.Update(stats.FileCache.Misses)
d.blockCacheHitGauge.Update(stats.BlockCache.Hits)
d.blockCacheMissGauge.Update(stats.BlockCache.Misses)
d.filterHitGauge.Update(stats.Filter.Hits)
@ -619,7 +639,7 @@ func (d *Database) meter(refresh time.Duration, namespace string) {
if i >= len(d.levelsGauge) {
d.levelsGauge = append(d.levelsGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("tables/level%v", i), nil))
}
d.levelsGauge[i].Update(level.NumFiles)
d.levelsGauge[i].Update(level.TablesCount)
}
// Sleep a bit, then repeat the stats collection

59
ethdb/pebble/pebble_test.go
View file

@ -21,7 +21,11 @@ import (
"testing"
"github.com/cockroachdb/pebble"
pebblev1 "github.com/cockroachdb/pebble"
pebblev2 "github.com/cockroachdb/pebble/v2"
vfsv2 "github.com/cockroachdb/pebble/v2/vfs"
"github.com/cockroachdb/pebble/vfs"
vfsv1 "github.com/cockroachdb/pebble/vfs"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/ethdb/dbtest"
)
@ -29,8 +33,8 @@ import (
func TestPebbleDB(t *testing.T) {
t.Run("DatabaseSuite", func(t *testing.T) {
dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore {
db, err := pebble.Open("", &pebble.Options{
FS: vfs.NewMem(),
db, err := pebblev2.Open("", &pebblev2.Options{
FS: vfsv2.NewMem(),
})
if err != nil {
t.Fatal(err)
@ -39,13 +43,24 @@ func TestPebbleDB(t *testing.T) {
db: db,
}
})
dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore {
db, err := pebblev1.Open("", &pebblev1.Options{
FS: vfsv1.NewMem(),
})
if err != nil {
t.Fatal(err)
}
return &V1Database{
db: db,
}
})
})
}
func BenchmarkPebbleDB(b *testing.B) {
dbtest.BenchDatabaseSuite(b, func() ethdb.KeyValueStore {
db, err := pebble.Open("", &pebble.Options{
FS: vfs.NewMem(),
db, err := pebblev2.Open("", &pebblev2.Options{
FS: vfsv2.NewMem(),
})
if err != nil {
b.Fatal(err)
@ -54,9 +69,20 @@ func BenchmarkPebbleDB(b *testing.B) {
db: db,
}
})
dbtest.BenchDatabaseSuite(b, func() ethdb.KeyValueStore {
db, err := pebblev1.Open("", &pebblev1.Options{
FS: vfsv1.NewMem(),
})
if err != nil {
b.Fatal(err)
}
return &V1Database{
db: db,
}
})
}
func TestPebbleLogData(t *testing.T) {
func TestPebbleLogDataV1(t *testing.T) {
db, err := pebble.Open("", &pebble.Options{
FS: vfs.NewMem(),
})
@ -78,3 +104,26 @@ func TestPebbleLogData(t *testing.T) {
t.Fatal("Unknown database entry")
}
}
func TestPebbleLogDataV2(t *testing.T) {
db, err := pebblev2.Open("", &pebblev2.Options{
FS: vfsv2.NewMem(),
})
if err != nil {
t.Fatal(err)
}
_, _, err = db.Get(nil)
if !errors.Is(err, pebblev2.ErrNotFound) {
t.Fatal("Unknown database entry")
}
b := db.NewBatch()
b.LogData(nil, nil)
db.Apply(b, pebblev2.Sync)
_, _, err = db.Get(nil)
if !errors.Is(err, pebblev2.ErrNotFound) {
t.Fatal("Unknown database entry")
}
}

753
ethdb/pebble/pebble_v1.go Normal file
View file

@ -0,0 +1,753 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Legacy pebble v1 wrapper. This file mirrors pebble.go but with V1-prefixed
// types so that it can coexist alongside a future v2 variant in the same package.
package pebble
import (
"errors"
"fmt"
"runtime"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/cockroachdb/pebble"
"github.com/cockroachdb/pebble/bloom"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
// V1Database is a persistent key-value store based on the pebble v1 storage engine.
// Apart from basic data storage functionality it also supports batch writes and
// iterating over the keyspace in binary-alphabetical order.
type V1Database struct {
fn string // filename for reporting
db *pebble.DB // Underlying pebble storage engine
namespace string // Namespace for metrics
compTimeMeter *metrics.Meter // Meter for measuring the total time spent in database compaction
compReadMeter *metrics.Meter // Meter for measuring the data read during compaction
compWriteMeter *metrics.Meter // Meter for measuring the data written during compaction
writeDelayNMeter *metrics.Meter // Meter for measuring the write delay number due to database compaction
writeDelayMeter *metrics.Meter // Meter for measuring the write delay duration due to database compaction
diskSizeGauge *metrics.Gauge // Gauge for tracking the size of all the levels in the database
diskReadMeter *metrics.Meter // Meter for measuring the effective amount of data read
diskWriteMeter *metrics.Meter // Meter for measuring the effective amount of data written
memCompGauge *metrics.Gauge // Gauge for tracking the number of memory compaction
level0CompGauge *metrics.Gauge // Gauge for tracking the number of table compaction in level0
nonlevel0CompGauge *metrics.Gauge // Gauge for tracking the number of table compaction in non0 level
seekCompGauge *metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt
manualMemAllocGauge *metrics.Gauge // Gauge for tracking amount of non-managed memory currently allocated
liveMemTablesGauge *metrics.Gauge // Gauge for tracking the number of live memory tables
zombieMemTablesGauge *metrics.Gauge // Gauge for tracking the number of zombie memory tables
blockCacheHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in the block cache
blockCacheMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in the block cache
tableCacheHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in the table cache
tableCacheMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in the table cache
filterHitGauge *metrics.Gauge // Gauge for tracking the number of total hit in bloom filter
filterMissGauge *metrics.Gauge // Gauge for tracking the number of total miss in bloom filter
estimatedCompDebtGauge *metrics.Gauge // Gauge for tracking the number of bytes that need to be compacted
liveCompGauge *metrics.Gauge // Gauge for tracking the number of in-progress compactions
liveCompSizeGauge *metrics.Gauge // Gauge for tracking the size of in-progress compactions
liveIterGauge *metrics.Gauge // Gauge for tracking the number of live database iterators
levelsGauge []*metrics.Gauge // Gauge for tracking the number of tables in levels
quitLock sync.RWMutex // Mutex protecting the quit channel and the closed flag
quitChan chan chan error // Quit channel to stop the metrics collection before closing the database
closed bool // keep track of whether we're Closed
log log.Logger // Contextual logger tracking the database path
activeComp int // Current number of active compactions
compStartTime time.Time // The start time of the earliest currently-active compaction
compTime atomic.Int64 // Total time spent in compaction in ns
level0Comp atomic.Uint32 // Total number of level-zero compactions
nonLevel0Comp atomic.Uint32 // Total number of non level-zero compactions
writeStalled atomic.Bool // Flag whether the write is stalled
writeDelayStartTime time.Time // The start time of the latest write stall
writeDelayReason string // The reason of the latest write stall
writeDelayCount atomic.Int64 // Total number of write stall counts
writeDelayTime atomic.Int64 // Total time spent in write stalls
writeOptions *pebble.WriteOptions
}
func (d *V1Database) onCompactionBegin(info pebble.CompactionInfo) {
if d.activeComp == 0 {
d.compStartTime = time.Now()
}
l0 := info.Input[0]
if l0.Level == 0 {
d.level0Comp.Add(1)
} else {
d.nonLevel0Comp.Add(1)
}
d.activeComp++
}
func (d *V1Database) onCompactionEnd(info pebble.CompactionInfo) {
if d.activeComp == 1 {
d.compTime.Add(int64(time.Since(d.compStartTime)))
} else if d.activeComp == 0 {
panic("should not happen")
}
d.activeComp--
}
func (d *V1Database) onWriteStallBegin(b pebble.WriteStallBeginInfo) {
d.writeDelayStartTime = time.Now()
d.writeDelayCount.Add(1)
d.writeStalled.Store(true)
// Take just the first word of the reason. These are two potential
// reasons for the write stall:
// - memtable count limit reached
// - L0 file count limit exceeded
reason := b.Reason
if i := strings.IndexByte(reason, ' '); i != -1 {
reason = reason[:i]
}
if reason == "L0" || reason == "memtable" {
d.writeDelayReason = reason
metrics.GetOrRegisterGauge(d.namespace+"stall/count/"+reason, nil).Inc(1)
}
}
func (d *V1Database) onWriteStallEnd() {
d.writeDelayTime.Add(int64(time.Since(d.writeDelayStartTime)))
d.writeStalled.Store(false)
if d.writeDelayReason != "" {
metrics.GetOrRegisterResettingTimer(d.namespace+"stall/time/"+d.writeDelayReason, nil).UpdateSince(d.writeDelayStartTime)
d.writeDelayReason = ""
}
d.writeDelayStartTime = time.Time{}
}
// NewV1 returns a wrapped pebble v1 DB object. The namespace is the prefix that the
// metrics reporting should use for surfacing internal stats.
func NewV1(file string, cache int, handles int, namespace string, readonly bool) (*V1Database, error) {
// Ensure we have some minimal caching and file guarantees
if cache < minCache {
cache = minCache
}
if handles < minHandles {
handles = minHandles
}
logger := log.New("database", file)
logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles, "version", "v1")
// The max memtable size is limited by the uint32 offsets stored in
// internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry.
//
// - MaxUint32 on 64-bit platforms;
// - MaxInt on 32-bit platforms.
//
// It is used when slices are limited to Uint32 on 64-bit platforms (the
// length limit for slices is naturally MaxInt on 32-bit platforms).
//
// Taken from https://github.com/cockroachdb/pebble/blob/master/internal/constants/constants.go
maxMemTableSize := (1<<31)<<(^uint(0)>>63) - 1
// Four memory tables are configured, each with a default size of 256 MB.
// Having multiple smaller memory tables while keeping the total memory
// limit unchanged allows writes to be flushed more smoothly. This helps
// avoid compaction spikes and mitigates write stalls caused by heavy
// compaction workloads.
memTableNumber := 4
memTableSize := cache * 1024 * 1024 / 2 / memTableNumber
// The memory table size is currently capped at maxMemTableSize-1 due to a
// known bug in the pebble where maxMemTableSize is not recognized as a
// valid size.
//
// TODO use the maxMemTableSize as the maximum table size once the issue
// in pebble is fixed.
if memTableSize >= maxMemTableSize {
memTableSize = maxMemTableSize - 1
}
db := &V1Database{
fn: file,
log: logger,
quitChan: make(chan chan error),
namespace: namespace,
// Use asynchronous write mode by default. Otherwise, the overhead of frequent fsync
// operations can be significant, especially on platforms with slow fsync performance
// (e.g., macOS) or less capable SSDs.
//
// Note that enabling async writes means recent data may be lost in the event of an
// application-level panic (writes will also be lost on a machine-level failure,
// of course). Geth is expected to handle recovery from an unclean shutdown.
writeOptions: pebble.NoSync,
}
opt := &pebble.Options{
// Pebble has a single combined cache area and the write
// buffers are taken from this too. Assign all available
// memory allowance for cache.
Cache: pebble.NewCache(int64(cache * 1024 * 1024)),
MaxOpenFiles: handles,
// The size of memory table(as well as the write buffer).
// Note, there may have more than two memory tables in the system.
MemTableSize: uint64(memTableSize),
// MemTableStopWritesThreshold places a hard limit on the number
// of the existent MemTables(including the frozen one).
//
// Note, this must be the number of tables not the size of all memtables
// according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742
// and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903.
//
// MemTableStopWritesThreshold is set to twice the maximum number of
// allowed memtables to accommodate temporary spikes.
MemTableStopWritesThreshold: memTableNumber * 2,
// The default compaction concurrency(1 thread),
// Here use all available CPUs for faster compaction.
MaxConcurrentCompactions: runtime.NumCPU,
// Per-level options. Options for at least one level must be specified. The
// options for the last level are used for all subsequent levels.
Levels: []pebble.LevelOptions{
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 4 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 8 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 16 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 32 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
{TargetFileSize: 64 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)},
// Pebble doesn't use the Bloom filter at level6 for read efficiency.
{TargetFileSize: 128 * 1024 * 1024},
},
ReadOnly: readonly,
EventListener: &pebble.EventListener{
CompactionBegin: db.onCompactionBegin,
CompactionEnd: db.onCompactionEnd,
WriteStallBegin: db.onWriteStallBegin,
WriteStallEnd: db.onWriteStallEnd,
},
Logger: panicLogger{}, // TODO(karalabe): Delete when this is upstreamed in Pebble
// Pebble is configured to use asynchronous write mode, meaning write operations
// return as soon as the data is cached in memory, without waiting for the WAL
// to be written. This mode offers better write performance but risks losing
// recent writes if the application crashes or a power failure/system crash occurs.
//
// By setting the WALBytesPerSync, the cached WAL writes will be periodically
// flushed at the background if the accumulated size exceeds this threshold.
WALBytesPerSync: 5 * ethdb.IdealBatchSize,
// L0CompactionThreshold specifies the number of L0 read-amplification
// necessary to trigger an L0 compaction. It essentially refers to the
// number of sub-levels at the L0. For each sub-level, it contains several
// L0 files which are non-overlapping with each other, typically produced
// by a single memory-table flush.
//
// The default value in Pebble is 4, which is a bit too large to have
// the compaction debt as around 10GB. By reducing it to 2, the compaction
// debt will be less than 1GB, but with more frequent compactions scheduled.
L0CompactionThreshold: 2,
}
// Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130
// for more details.
opt.Experimental.ReadSamplingMultiplier = -1
// These two settings define the conditions under which compaction concurrency
// is increased. Specifically, one additional compaction job will be enabled when:
// - there is one more overlapping sub-level0;
// - there is an additional 256 MB of compaction debt;
//
// The maximum concurrency is still capped by MaxConcurrentCompactions, but with
// these settings compactions can scale up more readily.
opt.Experimental.L0CompactionConcurrency = 1
opt.Experimental.CompactionDebtConcurrency = 1 << 28 // 256MB
// Open the db and recover any potential corruptions
innerDB, err := pebble.Open(file, opt)
if err != nil {
return nil, err
}
db.db = innerDB
db.compTimeMeter = metrics.GetOrRegisterMeter(namespace+"compact/time", nil)
db.compReadMeter = metrics.GetOrRegisterMeter(namespace+"compact/input", nil)
db.compWriteMeter = metrics.GetOrRegisterMeter(namespace+"compact/output", nil)
db.diskSizeGauge = metrics.GetOrRegisterGauge(namespace+"disk/size", nil)
db.diskReadMeter = metrics.GetOrRegisterMeter(namespace+"disk/read", nil)
db.diskWriteMeter = metrics.GetOrRegisterMeter(namespace+"disk/write", nil)
db.writeDelayMeter = metrics.GetOrRegisterMeter(namespace+"compact/writedelay/duration", nil)
db.writeDelayNMeter = metrics.GetOrRegisterMeter(namespace+"compact/writedelay/counter", nil)
db.memCompGauge = metrics.GetOrRegisterGauge(namespace+"compact/memory", nil)
db.level0CompGauge = metrics.GetOrRegisterGauge(namespace+"compact/level0", nil)
db.nonlevel0CompGauge = metrics.GetOrRegisterGauge(namespace+"compact/nonlevel0", nil)
db.seekCompGauge = metrics.GetOrRegisterGauge(namespace+"compact/seek", nil)
db.manualMemAllocGauge = metrics.GetOrRegisterGauge(namespace+"memory/manualalloc", nil)
db.liveMemTablesGauge = metrics.GetOrRegisterGauge(namespace+"table/live", nil)
db.zombieMemTablesGauge = metrics.GetOrRegisterGauge(namespace+"table/zombie", nil)
db.blockCacheHitGauge = metrics.GetOrRegisterGauge(namespace+"cache/block/hit", nil)
db.blockCacheMissGauge = metrics.GetOrRegisterGauge(namespace+"cache/block/miss", nil)
db.tableCacheHitGauge = metrics.GetOrRegisterGauge(namespace+"cache/table/hit", nil)
db.tableCacheMissGauge = metrics.GetOrRegisterGauge(namespace+"cache/table/miss", nil)
db.filterHitGauge = metrics.GetOrRegisterGauge(namespace+"filter/hit", nil)
db.filterMissGauge = metrics.GetOrRegisterGauge(namespace+"filter/miss", nil)
db.estimatedCompDebtGauge = metrics.GetOrRegisterGauge(namespace+"compact/estimateDebt", nil)
db.liveCompGauge = metrics.GetOrRegisterGauge(namespace+"compact/live/count", nil)
db.liveCompSizeGauge = metrics.GetOrRegisterGauge(namespace+"compact/live/size", nil)
db.liveIterGauge = metrics.GetOrRegisterGauge(namespace+"iter/count", nil)
// Start up the metrics gathering and return
go db.meter(metricsGatheringInterval, namespace)
return db, nil
}
// Close stops the metrics collection, flushes any pending data to disk and closes
// all io accesses to the underlying key-value store.
func (d *V1Database) Close() error {
d.quitLock.Lock()
defer d.quitLock.Unlock()
// Allow double closing, simplifies things
if d.closed {
return nil
}
d.closed = true
if d.quitChan != nil {
errc := make(chan error)
d.quitChan <- errc
if err := <-errc; err != nil {
d.log.Error("Metrics collection failed", "err", err)
}
d.quitChan = nil
}
return d.db.Close()
}
// Has retrieves if a key is present in the key-value store.
func (d *V1Database) Has(key []byte) (bool, error) {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return false, pebble.ErrClosed
}
_, closer, err := d.db.Get(key)
if err == pebble.ErrNotFound {
return false, nil
} else if err != nil {
return false, err
}
if err = closer.Close(); err != nil {
return false, err
}
return true, nil
}
// Get retrieves the given key if it's present in the key-value store.
func (d *V1Database) Get(key []byte) ([]byte, error) {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return nil, pebble.ErrClosed
}
dat, closer, err := d.db.Get(key)
if err != nil {
return nil, err
}
ret := make([]byte, len(dat))
copy(ret, dat)
if err = closer.Close(); err != nil {
return nil, err
}
return ret, nil
}
// Put inserts the given value into the key-value store.
func (d *V1Database) Put(key []byte, value []byte) error {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return pebble.ErrClosed
}
return d.db.Set(key, value, d.writeOptions)
}
// Delete removes the key from the key-value store.
func (d *V1Database) Delete(key []byte) error {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return pebble.ErrClosed
}
return d.db.Delete(key, d.writeOptions)
}
// DeleteRange deletes all of the keys (and values) in the range [start,end)
// (inclusive on start, exclusive on end).
func (d *V1Database) DeleteRange(start, end []byte) error {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return pebble.ErrClosed
}
// There is no special flag to represent the end of key range
// in pebble(nil in leveldb). Use an ugly hack to construct a
// large key to represent it.
if end == nil {
end = ethdb.MaximumKey
}
return d.db.DeleteRange(start, end, d.writeOptions)
}
// NewBatch creates a write-only key-value store that buffers changes to its host
// database until a final write is called.
func (d *V1Database) NewBatch() ethdb.Batch {
return &v1batch{
b: d.db.NewBatch(),
db: d,
}
}
// NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
func (d *V1Database) NewBatchWithSize(size int) ethdb.Batch {
return &v1batch{
b: d.db.NewBatchWithSize(size),
db: d,
}
}
// Stat returns the internal metrics of Pebble in a text format. It's a developer
// method to read everything there is to read, independent of Pebble version.
func (d *V1Database) Stat() (string, error) {
return d.db.Metrics().String(), nil
}
// Compact flattens the underlying data store for the given key range. In essence,
// deleted and overwritten versions are discarded, and the data is rearranged to
// reduce the cost of operations needed to access them.
//
// A nil start is treated as a key before all keys in the data store; a nil limit
// is treated as a key after all keys in the data store. If both is nil then it
// will compact entire data store.
func (d *V1Database) Compact(start []byte, limit []byte) error {
// There is no special flag to represent the end of key range
// in pebble(nil in leveldb). Use an ugly hack to construct a
// large key to represent it.
// Note any prefixed database entry will be smaller than this
// flag, as for trie nodes we need the 32 byte 0xff because
// there might be a shared prefix starting with a number of
// 0xff-s, so 32 ensures than only a hash collision could touch it.
// https://github.com/cockroachdb/pebble/issues/2359#issuecomment-1443995833
if limit == nil {
limit = ethdb.MaximumKey
}
return d.db.Compact(start, limit, true) // Parallelization is preferred
}
// Path returns the path to the database directory.
func (d *V1Database) Path() string {
return d.fn
}
// SyncKeyValue flushes all pending writes in the write-ahead-log to disk,
// ensuring data durability up to that point.
func (d *V1Database) SyncKeyValue() error {
// The entry (value=nil) is not written to the database; it is only
// added to the WAL. Writing this special log entry in sync mode
// automatically flushes all previous writes, ensuring database
// durability up to this point.
b := d.db.NewBatch()
b.LogData(nil, nil)
return d.db.Apply(b, pebble.Sync)
}
// NewIterator creates a binary-alphabetical iterator over a subset
// of database content with a particular key prefix, starting at a particular
// initial key (or after, if it does not exist).
func (d *V1Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator {
iter, _ := d.db.NewIter(&pebble.IterOptions{
LowerBound: append(prefix, start...),
UpperBound: upperBound(prefix),
})
iter.First()
return &v1pebbleIterator{iter: iter, moved: true, released: false}
}
// meter periodically retrieves internal pebble counters and reports them to
// the metrics subsystem.
func (d *V1Database) meter(refresh time.Duration, namespace string) {
var errc chan error
timer := time.NewTimer(refresh)
defer timer.Stop()
// Create storage and warning log tracer for write delay.
var (
compTimes [2]int64
compWrites [2]int64
compReads [2]int64
nWrites [2]int64
writeDelayTimes [2]int64
writeDelayCounts [2]int64
lastWriteStallReport time.Time
)
// Iterate ad infinitum and collect the stats
for i := 1; errc == nil; i++ {
var (
compWrite int64
compRead int64
nWrite int64
stats = d.db.Metrics()
compTime = d.compTime.Load()
writeDelayCount = d.writeDelayCount.Load()
writeDelayTime = d.writeDelayTime.Load()
nonLevel0CompCount = int64(d.nonLevel0Comp.Load())
level0CompCount = int64(d.level0Comp.Load())
)
writeDelayTimes[i%2] = writeDelayTime
writeDelayCounts[i%2] = writeDelayCount
compTimes[i%2] = compTime
for _, levelMetrics := range stats.Levels {
nWrite += int64(levelMetrics.BytesCompacted)
nWrite += int64(levelMetrics.BytesFlushed)
compWrite += int64(levelMetrics.BytesCompacted)
compRead += int64(levelMetrics.BytesRead)
}
nWrite += int64(stats.WAL.BytesWritten)
compWrites[i%2] = compWrite
compReads[i%2] = compRead
nWrites[i%2] = nWrite
d.writeDelayNMeter.Mark(writeDelayCounts[i%2] - writeDelayCounts[(i-1)%2])
d.writeDelayMeter.Mark(writeDelayTimes[i%2] - writeDelayTimes[(i-1)%2])
// Print a warning log if writing has been stalled for a while. The log will
// be printed per minute to avoid overwhelming users.
if d.writeStalled.Load() && writeDelayCounts[i%2] == writeDelayCounts[(i-1)%2] &&
time.Now().After(lastWriteStallReport.Add(degradationWarnInterval)) {
d.log.Warn("Database compacting, degraded performance")
lastWriteStallReport = time.Now()
}
d.compTimeMeter.Mark(compTimes[i%2] - compTimes[(i-1)%2])
d.compReadMeter.Mark(compReads[i%2] - compReads[(i-1)%2])
d.compWriteMeter.Mark(compWrites[i%2] - compWrites[(i-1)%2])
d.diskSizeGauge.Update(int64(stats.DiskSpaceUsage()))
d.diskReadMeter.Mark(0) // pebble doesn't track non-compaction reads
d.diskWriteMeter.Mark(nWrites[i%2] - nWrites[(i-1)%2])
// See https://github.com/cockroachdb/pebble/pull/1628#pullrequestreview-1026664054
manuallyAllocated := stats.BlockCache.Size + int64(stats.MemTable.Size) + int64(stats.MemTable.ZombieSize)
d.manualMemAllocGauge.Update(manuallyAllocated)
d.memCompGauge.Update(stats.Flush.Count)
d.nonlevel0CompGauge.Update(nonLevel0CompCount)
d.level0CompGauge.Update(level0CompCount)
d.seekCompGauge.Update(stats.Compact.ReadCount)
d.liveCompGauge.Update(stats.Compact.NumInProgress)
d.liveCompSizeGauge.Update(stats.Compact.InProgressBytes)
d.liveIterGauge.Update(stats.TableIters)
d.liveMemTablesGauge.Update(stats.MemTable.Count)
d.zombieMemTablesGauge.Update(stats.MemTable.ZombieCount)
d.estimatedCompDebtGauge.Update(int64(stats.Compact.EstimatedDebt))
d.tableCacheHitGauge.Update(stats.TableCache.Hits)
d.tableCacheMissGauge.Update(stats.TableCache.Misses)
d.blockCacheHitGauge.Update(stats.BlockCache.Hits)
d.blockCacheMissGauge.Update(stats.BlockCache.Misses)
d.filterHitGauge.Update(stats.Filter.Hits)
d.filterMissGauge.Update(stats.Filter.Misses)
for i, level := range stats.Levels {
// Append metrics for additional layers
if i >= len(d.levelsGauge) {
d.levelsGauge = append(d.levelsGauge, metrics.GetOrRegisterGauge(namespace+fmt.Sprintf("tables/level%v", i), nil))
}
d.levelsGauge[i].Update(level.NumFiles)
}
// Sleep a bit, then repeat the stats collection
select {
case errc = <-d.quitChan:
// Quit requesting, stop hammering the database
case <-timer.C:
timer.Reset(refresh)
// Timeout, gather a new set of stats
}
}
errc <- nil
}
// v1batch is a write-only batch that commits changes to its host database
// when Write is called. A v1batch cannot be used concurrently.
type v1batch struct {
b *pebble.Batch
db *V1Database
size int
}
// Put inserts the given value into the batch for later committing.
func (b *v1batch) Put(key, value []byte) error {
if err := b.b.Set(key, value, nil); err != nil {
return err
}
b.size += len(key) + len(value)
return nil
}
// Delete inserts the key removal into the batch for later committing.
func (b *v1batch) Delete(key []byte) error {
if err := b.b.Delete(key, nil); err != nil {
return err
}
b.size += len(key)
return nil
}
// DeleteRange removes all keys in the range [start, end) from the batch for
// later committing, inclusive on start, exclusive on end.
func (b *v1batch) DeleteRange(start, end []byte) error {
// There is no special flag to represent the end of key range
// in pebble(nil in leveldb). Use an ugly hack to construct a
// large key to represent it.
if end == nil {
end = ethdb.MaximumKey
}
if err := b.b.DeleteRange(start, end, nil); err != nil {
return err
}
// Approximate size impact - just the keys
b.size += len(start) + len(end)
return nil
}
// ValueSize retrieves the amount of data queued up for writing.
func (b *v1batch) ValueSize() int {
return b.size
}
// Write flushes any accumulated data to disk.
func (b *v1batch) Write() error {
b.db.quitLock.RLock()
defer b.db.quitLock.RUnlock()
if b.db.closed {
return pebble.ErrClosed
}
return b.b.Commit(b.db.writeOptions)
}
// Reset resets the batch for reuse.
func (b *v1batch) Reset() {
b.b.Reset()
b.size = 0
}
// Replay replays the batch contents.
func (b *v1batch) Replay(w ethdb.KeyValueWriter) error {
reader := b.b.Reader()
for {
kind, k, v, ok, err := reader.Next()
if !ok || err != nil {
return err
}
// The (k,v) slices might be overwritten if the batch is reset/reused,
// and the receiver should copy them if they are to be retained long-term.
if kind == pebble.InternalKeyKindSet {
if err = w.Put(k, v); err != nil {
return err
}
} else if kind == pebble.InternalKeyKindDelete {
if err = w.Delete(k); err != nil {
return err
}
} else if kind == pebble.InternalKeyKindRangeDelete {
// For range deletion, k is the start key and v is the end key
if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok {
if err = rangeDeleter.DeleteRange(k, v); err != nil {
return err
}
} else {
return errors.New("ethdb.KeyValueWriter does not implement DeleteRange")
}
} else {
return fmt.Errorf("unhandled operation, keytype: %v", kind)
}
}
}
// Close closes the batch and releases all associated resources. After it is
// closed, any subsequent operations on this batch are undefined.
func (b *v1batch) Close() {
b.b.Close()
}
// v1pebbleIterator is a wrapper of underlying iterator in storage engine.
// The purpose of this structure is to implement the missing APIs.
//
// The v1pebbleIterator is not thread-safe.
type v1pebbleIterator struct {
iter *pebble.Iterator
moved bool
released bool
}
// Next moves the iterator to the next key/value pair. It returns whether the
// iterator is exhausted.
func (iter *v1pebbleIterator) Next() bool {
if iter.moved {
iter.moved = false
return iter.iter.Valid()
}
return iter.iter.Next()
}
// Error returns any accumulated error. Exhausting all the key/value pairs
// is not considered to be an error.
func (iter *v1pebbleIterator) Error() error {
return iter.iter.Error()
}
// Key returns the key of the current key/value pair, or nil if done. The caller
// should not modify the contents of the returned slice, and its contents may
// change on the next call to Next.
func (iter *v1pebbleIterator) Key() []byte {
return iter.iter.Key()
}
// Value returns the value of the current key/value pair, or nil if done. The
// caller should not modify the contents of the returned slice, and its contents
// may change on the next call to Next.
func (iter *v1pebbleIterator) Value() []byte {
return iter.iter.Value()
}
// Release releases associated resources. Release should always succeed and can
// be called multiple times without causing error.
func (iter *v1pebbleIterator) Release() {
if !iter.released {
iter.iter.Close()
iter.released = true
}
}

146
ethdb/pebble/version.go Normal file
View file

@ -0,0 +1,146 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pebble
import (
"fmt"
"runtime"
pebblev1 "github.com/cockroachdb/pebble"
v1bloom "github.com/cockroachdb/pebble/bloom"
pebblev2 "github.com/cockroachdb/pebble/v2"
v2vfs "github.com/cockroachdb/pebble/v2/vfs"
v1vfs "github.com/cockroachdb/pebble/vfs"
"github.com/ethereum/go-ethereum/log"
)
// formatMinV2 is the minimum FormatMajorVersion supported by pebble v2.
// Databases with a lower format version must be opened with pebble v1.
const formatMinV2 = pebblev2.FormatFlushableIngest
// PeekFormatVersion reads the format version of an existing pebble database
// without opening it.
func PeekFormatVersion(file string) (bool, uint64, error) {
desc, err := pebblev2.Peek(file, v2vfs.Default)
if err == nil && desc.Exists {
return true, uint64(desc.FormatMajorVersion), nil
}
// Pebble v2 dropped support for the legacy FormatMostCompatible layout,
// which relies on the CURRENT file rather than a manifest marker.
//
// Databases created by older Geth (which never set FormatMajorVersion
// and therefore default to FormatMostCompatible) are not recognized by
// v2's Peek: it reports Exists=false with a nil error instead of failing.
// It may also fail outright on some old databases. In both cases fall
// back to v1's Peek, which still understands the CURRENT-file layout.
desc1, err1 := pebblev1.Peek(file, v1vfs.Default)
if err1 != nil {
// Surface the v2 error if there was one, otherwise the v1 error.
// Such as the folder is not existent, fs.ErrNotExist.
if err != nil {
return false, 0, err
}
return false, 0, err1
}
if !desc1.Exists {
// Neither version found a database; treat as a new/empty directory.
return false, 0, nil
}
return true, uint64(desc1.FormatMajorVersion), nil
}
// NeedsV1 returns true if the database at the given path requires pebble v1
// to open (format version too old for pebble v2).
func NeedsV1(file string) bool {
exists, ver, err := PeekFormatVersion(file)
if err != nil || !exists {
return false // New database or error; use v2
}
return pebblev2.FormatMajorVersion(ver) < formatMinV2
}
// Upgrade upgrades an existing pebble v1 database to be compatible with pebble v2.
// It opens the database with pebble v1 at its current format version, then uses
// RatchetFormatMajorVersion to migrate to FormatFlushableIngest (the minimum format
// version that pebble v2 supports).
//
// Notably, it's not an irreversible upgrade, the database can still be opened with
// legacy Geth binary.
func Upgrade(file string) error {
exists, ver, err := PeekFormatVersion(file)
if err != nil {
return err
}
if !exists {
return fmt.Errorf("pebble database not found at %s", file)
}
if pebblev2.FormatMajorVersion(ver) >= formatMinV2 {
log.Info("Database format already compatible with pebble v2", "version", ver)
return nil
}
// FormatFlushableIngest exists in both pebble v1 and pebble v2 and it serves
// as the natural bridge point: a v1 database can be ratcheted up to
// FormatFlushableIngest using pebble v1, and then pebble v2 can open it since
// that's its minimum supported format.
v1Target := pebblev1.FormatFlushableIngest
log.Info("Upgrading pebble database format via v1", "from", ver, "to", v1Target)
numCPU := runtime.NumCPU()
opt := &pebblev1.Options{
// Open at the current on-disk format version; do not request a
// higher version here so that the upgrade happens explicitly via
// RatchetFormatMajorVersion below.
MaxConcurrentCompactions: func() int { return numCPU },
Levels: []pebblev1.LevelOptions{
{TargetFileSize: 2 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 4 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 8 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 16 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 32 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 64 * 1024 * 1024, FilterPolicy: v1bloom.FilterPolicy(10)},
{TargetFileSize: 128 * 1024 * 1024},
},
Logger: panicLogger{},
}
db, err := pebblev1.Open(file, opt)
if err != nil {
return fmt.Errorf("failed to open database with pebble v1 for upgrade: %w", err)
}
if err := db.RatchetFormatMajorVersion(v1Target); err != nil {
db.Close()
return fmt.Errorf("failed to ratchet format version to %d: %w", v1Target, err)
}
if err := db.Close(); err != nil {
return fmt.Errorf("failed to close database after v1 upgrade: %w", err)
}
log.Info("Pebble v1 format upgrade complete, verifying v2 compatibility")
// Verify that pebble v2 can open the upgraded database.
opt2 := &pebblev2.Options{
Logger: panicLogger{},
FormatMajorVersion: formatMinV2,
}
db2, err := pebblev2.Open(file, opt2)
if err != nil {
return fmt.Errorf("failed to open database with pebble v2 after upgrade: %w", err)
}
if err := db2.Close(); err != nil {
return fmt.Errorf("failed to close database after v2 verification: %w", err)
}
log.Info("Pebble database format upgrade complete")
return nil
}

135
ethdb/pebble/version_test.go Normal file
View file

@ -0,0 +1,135 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pebble
import (
"testing"
pebblev1 "github.com/cockroachdb/pebble"
pebblev2 "github.com/cockroachdb/pebble/v2"
v2vfs "github.com/cockroachdb/pebble/v2/vfs"
v1vfs "github.com/cockroachdb/pebble/vfs"
)
// TestPeekFormatVersionLegacyV1 verifies that PeekFormatVersion correctly
// detects a legacy pebble v1 database written in the FormatMostCompatible
// layout. Older Geth never set FormatMajorVersion, so pebble v1 defaulted to
// FormatMostCompatible, which uses the CURRENT file rather than a manifest
// marker. Pebble v2's Peek does not understand this layout and reports
// Exists=false with a nil error, so PeekFormatVersion must fall back to v1.
func TestPeekFormatVersionLegacyV1(t *testing.T) {
dir := t.TempDir()
// Create a v1 database with default options (no FormatMajorVersion set),
// which yields FormatMostCompatible, exactly as legacy Geth would.
db, err := pebblev1.Open(dir, &pebblev1.Options{})
if err != nil {
t.Fatalf("failed to create v1 database: %v", err)
}
if got := db.FormatMajorVersion(); got != pebblev1.FormatMostCompatible {
db.Close()
t.Fatalf("unexpected on-disk format version: have %d, want %d", got, pebblev1.FormatMostCompatible)
}
if err := db.Set([]byte("foo"), []byte("bar"), pebblev1.Sync); err != nil {
db.Close()
t.Fatalf("failed to write to v1 database: %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close v1 database: %v", err)
}
// Document the underlying pebble v2 behavior that motivates the v1
// fallback: v2's Peek silently fails to recognize this database.
if desc, err := pebblev2.Peek(dir, v2vfs.Default); err == nil && desc.Exists {
t.Fatal("expected pebble v2 Peek to not recognize a FormatMostCompatible database")
}
exists, ver, err := PeekFormatVersion(dir)
if err != nil {
t.Fatalf("PeekFormatVersion returned error: %v", err)
}
if !exists {
t.Fatal("expected legacy v1 database to be detected, got exists=false")
}
if ver != uint64(pebblev1.FormatMostCompatible) {
t.Fatalf("unexpected format version: have %d, want %d", ver, pebblev1.FormatMostCompatible)
}
// The database is too old for pebble v2, so it must be routed through v1.
if !NeedsV1(dir) {
t.Fatal("expected NeedsV1 to be true for a FormatMostCompatible database")
}
}
// TestPeekFormatVersionV2 verifies that PeekFormatVersion detects a database
// written at a pebble v2 compatible format version directly via v2's Peek.
func TestPeekFormatVersionV2(t *testing.T) {
dir := t.TempDir()
db, err := pebblev2.Open(dir, &pebblev2.Options{
FormatMajorVersion: formatMinV2,
})
if err != nil {
t.Fatalf("failed to create v2 database: %v", err)
}
if err := db.Set([]byte("foo"), []byte("bar"), pebblev2.Sync); err != nil {
db.Close()
t.Fatalf("failed to write to v2 database: %v", err)
}
if err := db.Close(); err != nil {
t.Fatalf("failed to close v2 database: %v", err)
}
exists, ver, err := PeekFormatVersion(dir)
if err != nil {
t.Fatalf("PeekFormatVersion returned error: %v", err)
}
if !exists {
t.Fatal("expected v2 database to be detected, got exists=false")
}
if ver != uint64(formatMinV2) {
t.Fatalf("unexpected format version: have %d, want %d", ver, formatMinV2)
}
if NeedsV1(dir) {
t.Fatal("expected NeedsV1 to be false for a v2 database")
}
}
// TestPeekFormatVersionEmpty verifies that an empty directory (a new database
// location) is reported as non-existent by both v2 and v1 Peek, rather than
// being misreported.
func TestPeekFormatVersionEmpty(t *testing.T) {
dir := t.TempDir()
// Sanity check that v1's Peek also reports a non-existent database here,
// so the test exercises the "neither version found a database" branch.
if desc, err := pebblev1.Peek(dir, v1vfs.Default); err != nil {
t.Fatalf("v1 Peek on empty directory returned error: %v", err)
} else if desc.Exists {
t.Fatal("expected v1 Peek to report no database in an empty directory")
}
exists, _, err := PeekFormatVersion(dir)
if err != nil {
t.Fatalf("PeekFormatVersion returned error: %v", err)
}
if exists {
t.Fatal("expected no database in an empty directory, got exists=true")
}
if NeedsV1(dir) {
t.Fatal("expected NeedsV1 to be false for an empty directory")
}
}

14
go.mod
View file

@ -13,6 +13,7 @@ require (
github.com/cespare/cp v0.1.0
github.com/cloudflare/cloudflare-go v0.114.0
github.com/cockroachdb/pebble v1.1.5
github.com/cockroachdb/pebble/v2 v2.1.4
github.com/consensys/gnark-crypto v0.18.1
github.com/crate-crypto/go-eth-kzg v1.5.0
github.com/davecgh/go-spew v1.1.1
@ -45,7 +46,7 @@ require (
github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c
github.com/jackpal/go-nat-pmp v1.0.2
github.com/jedisct1/go-minisign v0.0.0-20230811132847-661be99b8267
github.com/klauspost/compress v1.17.8
github.com/klauspost/compress v1.17.11
github.com/kylelemons/godebug v1.1.0
github.com/mattn/go-colorable v0.1.13
github.com/mattn/go-isatty v0.0.20
@ -83,10 +84,15 @@ require (
)
require (
github.com/RaduBerinde/axisds v0.1.0 // indirect
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect
github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect
github.com/pion/dtls/v3 v3.1.2 // indirect
github.com/pion/transport/v4 v4.0.1 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
@ -101,7 +107,7 @@ require (
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/DataDog/zstd v1.4.5 // indirect
github.com/DataDog/zstd v1.5.7 // indirect
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.13 // indirect
@ -153,10 +159,10 @@ require (
github.com/pion/logging v0.2.4 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.15.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect

38
go.sum
View file

@ -10,16 +10,22 @@ github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE=
github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU=
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI=
github.com/RaduBerinde/axisds v0.1.0 h1:YItk/RmU5nvlsv/awo2Fjx97Mfpt4JfgtEVAGPrLdz8=
github.com/RaduBerinde/axisds v0.1.0/go.mod h1:UHGJonU9z4YYGKJxSaC6/TNcLOBptpmM5m2Cksbnw0Y=
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 h1:bsU8Tzxr/PNz75ayvCnxKZWEYdLMPDkUgticP4a4Bvk=
github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54/go.mod h1:0tr7FllbE9gJkHq7CVeeDDFAFKQVy5RnCSSNBOvdqbc=
github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA=
github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
github.com/VictoriaMetrics/fastcache v1.13.0 h1:AW4mheMR5Vd9FkAPUv+NH6Nhw+fmbTMGMsNAoA/+4G0=
github.com/VictoriaMetrics/fastcache v1.13.0/go.mod h1:hHXhl4DA2fTL2HTZDJFXWgW0LNjo6B+4aj2Wmng3TjU=
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f h1:JjxwchlOepwsUWcQwD2mLUAGE9aCp0/ehy6yCHFBOvo=
github.com/aclements/go-perfevent v0.0.0-20240301234650-f7843625020f/go.mod h1:tMDTce/yLLN/SK8gMOxQfnyeMeCg8KGzp0D1cbECEeo=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
github.com/aws/aws-sdk-go-v2 v1.21.2 h1:+LXZ0sgo8quN9UOKXXzAWRT3FWd4NxeXWOZom9pE7GA=
@ -63,18 +69,26 @@ github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86c
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/cloudflare/cloudflare-go v0.114.0 h1:ucoti4/7Exo0XQ+rzpn1H+IfVVe++zgiM+tyKtf0HUA=
github.com/cloudflare/cloudflare-go v0.114.0/go.mod h1:O7fYfFfA6wKqKFn2QIR9lhj7FDw6VQCGOY6hd2TBtd0=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU=
github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac=
github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI=
github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5/go.mod h1:jsaKMvD3RBCATk1/jbUZM8C9idWBJME9+VRZ5+Liq1g=
github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895 h1:XANOgPYtvELQ/h4IrmPAohXqe2pWA8Bwhejr3VQoZsA=
github.com/cockroachdb/metamorphic v0.0.0-20231108215700-4ba948b56895/go.mod h1:aPd7gM9ov9M8v32Yy5NJrDyOcD8z642dqs+F0CeNXfA=
github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw=
github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo=
github.com/cockroachdb/pebble/v2 v2.1.4 h1:j9wPgMDbkErFdAKYFGhsoCcvzcjR+6zrJ4jhKtJ6bOk=
github.com/cockroachdb/pebble/v2 v2.1.4/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8=
github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI=
github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI=
@ -138,6 +152,8 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x
github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM=
github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs=
github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
@ -233,8 +249,8 @@ github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4
github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@ -272,6 +288,8 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw=
github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
@ -315,14 +333,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM=
github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8=
github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc=
github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg=
github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM=
github.com/protolambda/bls12-381-util v0.1.0 h1:05DU2wJN7DTU7z28+Q+zejXkIsA/MF8JZQGhtBZZiWk=
github.com/protolambda/bls12-381-util v0.1.0/go.mod h1:cdkysJTRpeFeuUVx/TXGDQNMTiRAalk1vQw3TYTHcE4=
github.com/protolambda/zrnt v0.34.1 h1:qW55rnhZJDnOb3TwFiFRJZi3yTXFrJdGOFQM7vCwYGg=

12
node/database.go
View file

@ -114,7 +114,19 @@ func newLevelDBDatabase(file string, cache int, handles int, namespace string, r
// newPebbleDBDatabase creates a persistent key-value database without a freezer
// moving immutable chain segments into cold storage.
//
// If the database already exists with a legacy pebble v1 format, it is opened
// using pebble v1 for backward compatibility and a warning is logged directing
// the user to upgrade offline. New databases use pebble v2.
func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.KeyValueStore, error) {
if pebble.NeedsV1(file) {
log.Warn("Pebble database uses legacy v1 format; upgrade offline with 'geth db pebble-upgrade'")
db, err := pebble.NewV1(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err
}
return db, nil
}
db, err := pebble.New(file, cache, handles, namespace, readonly)
if err != nil {
return nil, err