From bf0094bf43ebbe05cdbb1241aea685c2127124ee Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Thu, 11 Jun 2026 12:36:55 +0000 Subject: [PATCH 1/2] core/rawdb/eradb: integrate ere backend --- core/rawdb/eradb/eradb.go | 135 ++++++++++++------ core/rawdb/eradb/eradb_test.go | 100 +++++++++++++ .../sepolia-00000-deadbeef-noreceipts.ere | Bin 0 -> 222 bytes internal/era/execdb/reader.go | 6 + 4 files changed, 195 insertions(+), 46 deletions(-) create mode 100644 core/rawdb/eradb/testdata/noreceipts/sepolia-00000-deadbeef-noreceipts.ere diff --git a/core/rawdb/eradb/eradb.go b/core/rawdb/eradb/eradb.go index d715c824ed..390715d46f 100644 --- a/core/rawdb/eradb/eradb.go +++ b/core/rawdb/eradb/eradb.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package eradb implements a history backend using era1 files. +// Package eradb implements a history backend using era1 or ere files. package eradb import ( @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era/execdb" "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -36,7 +37,7 @@ const openFileLimit = 64 var errClosed = errors.New("era store is closed") -// Store manages read access to a directory of era1 files. +// Store manages read access to a directory of era1 or ere files. // The getter methods are thread-safe. type Store struct { datadir string @@ -52,7 +53,8 @@ type Store struct { type fileCacheEntry struct { refcount int // reference count. This is protected by Store.mu! opened chan struct{} // signals opening of file has completed - file *onedb.Era // the file + file era.Era // the file + slim bool // true if receipts are stored in the ere slim encoding err error // error from opening the file } @@ -77,7 +79,7 @@ func New(datadir string) (*Store, error) { return db, nil } -// Close closes all open era1 files in the cache. +// Close closes all open era files in the cache. func (db *Store) Close() { db.mu.Lock() defer db.mu.Unlock() @@ -132,12 +134,14 @@ func (db *Store) GetRawReceipts(number uint64) ([]byte, error) { if err != nil { return nil, err } - return convertReceipts(data) + return convertReceipts(data, entry.slim) } -// convertReceipts transforms an encoded block receipts list from the format -// used by era1 into the 'storage' format used by the go-ethereum ancients database. -func convertReceipts(input []byte) ([]byte, error) { +// convertReceipts transforms an encoded block receipts list into the 'storage' +// format used by the go-ethereum ancients database, i.e. a list of +// [status, gas-used, logs]. The input uses the era1 network encoding, or the +// ere slim encoding when slim is true. +func convertReceipts(input []byte, slim bool) ([]byte, error) { var ( out bytes.Buffer enc = rlp.NewEncoderBuffer(&out) @@ -148,32 +152,41 @@ func convertReceipts(input []byte) ([]byte, error) { } outerList := enc.List() for i := 0; blockListIter.Next(); i++ { - kind, content, _, err := rlp.Split(blockListIter.Value()) - if err != nil { - return nil, fmt.Errorf("receipt %d invalid: %v", i, err) - } - var receiptData []byte - switch kind { - case rlp.Byte: - return nil, fmt.Errorf("receipt %d is single byte", i) - case rlp.String: - // Typed receipt - skip type. - receiptData = content[1:] - case rlp.List: - // Legacy receipt + var ( + receiptData []byte + skip int + ) + if slim { + // Slim receipt is [tx-type, status, gas-used, logs]: skip the tx-type. receiptData = blockListIter.Value() + skip = 0 + } else { + // Era1 receipt is [status, gas-used, bloom, logs], prefixed by the + // tx type if non-legacy: skip the bloom. + kind, content, _, err := rlp.Split(blockListIter.Value()) + if err != nil { + return nil, fmt.Errorf("receipt %d invalid: %v", i, err) + } + switch kind { + case rlp.Byte: + return nil, fmt.Errorf("receipt %d is single byte", i) + case rlp.String: + // Typed receipt - skip type. + receiptData = content[1:] + case rlp.List: + // Legacy receipt + receiptData = blockListIter.Value() + } + skip = 2 } - // Convert data list. - // Input is [status, gas-used, bloom, logs] - // Output is [status, gas-used, logs], i.e. we need to skip the bloom. dataIter, err := rlp.NewListIterator(receiptData) if err != nil { return nil, fmt.Errorf("receipt %d has invalid data: %v", i, err) } innerList := enc.List() for field := 0; dataIter.Next(); field++ { - if field == 2 { - continue // skip bloom + if field == skip { + continue } enc.Write(dataIter.Value()) } @@ -202,11 +215,11 @@ func (db *Store) getEraByEpoch(epoch uint64) *fileCacheEntry { case fileIsNew: // Open the file and put it into the cache. - e, err := db.openEraFile(epoch) + e, slim, err := db.openEraFile(epoch) if err != nil { db.fileFailedToOpen(epoch, entry, err) } else { - db.fileOpened(epoch, entry, e) + db.fileOpened(epoch, entry, e, slim) } close(entry.opened) @@ -250,7 +263,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC } // fileOpened is called after an era file has been successfully opened. -func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) { +func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file era.Era, slim bool) { db.mu.Lock() defer db.mu.Unlock() @@ -267,6 +280,7 @@ func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era // Add it to the LRU. This may evict an existing item, which we have to close. entry.file = file + entry.slim = slim evictedEpoch, evictedEntry, _ := db.lru.Add3(epoch, entry) if evictedEntry != nil { evictedEntry.derefAndClose(evictedEpoch) @@ -283,32 +297,61 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error entry.err = err } -func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) { - // File name scheme is --. - glob := fmt.Sprintf("*-%05d-*.era1", epoch) - matches, err := filepath.Glob(filepath.Join(db.datadir, glob)) - if err != nil { - return nil, err +// openEraFile opens the era file of the given epoch. The second return value +// signals whether the receipts in the file use the ere slim encoding. +func (db *Store) openEraFile(epoch uint64) (era.Era, bool, error) { + // File name scheme is -- for era1 files and + // --(-)* for ere files. + var matches []string + for _, glob := range []string{ + fmt.Sprintf("*-%05d-*.era1", epoch), + fmt.Sprintf("*-%05d-*.ere", epoch), + } { + m, err := filepath.Glob(filepath.Join(db.datadir, glob)) + if err != nil { + return nil, false, err + } + matches = append(matches, m...) } if len(matches) > 1 { - return nil, fmt.Errorf("multiple era1 files found for epoch %d", epoch) + return nil, false, fmt.Errorf("multiple era files found for epoch %d: %v", epoch, matches) } if len(matches) == 0 { - return nil, fs.ErrNotExist + return nil, false, fs.ErrNotExist } filename := matches[0] - e, err := onedb.Open(filename) - if err != nil { - return nil, err + var ( + e era.Era + err error + slim = filepath.Ext(filename) == ".ere" + ) + if slim { + var f *execdb.Era + f, err = execdb.Open(filename) + if err != nil { + return nil, false, err + } + // Files written with the "noreceipts" profile cannot serve as a + // history backend, since the receipts cannot be retrieved. + if !f.HasReceipts() { + f.Close() + return nil, false, fmt.Errorf("ere file %s contains no receipts", filepath.Base(filename)) + } + e = f + } else { + e, err = onedb.Open(filename) + if err != nil { + return nil, false, err + } } // Sanity-check start block. - if e.Start()%uint64(era.MaxSize) != 0 { + if e.Start() != epoch*uint64(era.MaxSize) { e.Close() - return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize) + return nil, false, fmt.Errorf("era file %s has wrong start block %d for epoch %d", filepath.Base(filename), e.Start(), epoch) } - log.Debug("Opened era1 file", "epoch", epoch) - return e.(*onedb.Era), nil + log.Debug("Opened era file", "epoch", epoch, "name", filepath.Base(filename)) + return e, slim, nil } // doneWithFile signals that the caller has finished using a file. @@ -339,9 +382,9 @@ func (entry *fileCacheEntry) derefAndClose(epoch uint64) (closed bool) { closeErr := entry.file.Close() if closeErr == nil { - log.Debug("Closed era1 file", "epoch", epoch) + log.Debug("Closed era file", "epoch", epoch) } else { - log.Warn("Error closing era1 file", "epoch", epoch, "err", closeErr) + log.Warn("Error closing era file", "epoch", epoch, "err", closeErr) } return true } diff --git a/core/rawdb/eradb/eradb_test.go b/core/rawdb/eradb/eradb_test.go index 41047dbbe9..5c3c97204f 100644 --- a/core/rawdb/eradb/eradb_test.go +++ b/core/rawdb/eradb/eradb_test.go @@ -17,10 +17,15 @@ package eradb import ( + "math/big" + "os" + "path/filepath" "sync" "testing" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/execdb" + "github.com/ethereum/go-ethereum/internal/era/onedb" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -48,6 +53,101 @@ func TestEraDatabase(t *testing.T) { assert.Equal(t, 3, len(receipts), "receipts length mismatch") } +// TestEreDatabase checks that the store can serve bodies and receipts from a +// directory of ere files, and that the receipts returned are byte-identical to +// the ones derived from the equivalent era1 files. +func TestEreDatabase(t *testing.T) { + dir := t.TempDir() + convertEra1ToEre(t, "testdata/sepolia-00000-643a00f7.era1", dir, "sepolia", 0) + convertEra1ToEre(t, "testdata/sepolia-00021-b8814b14.era1", dir, "sepolia", 21) + + db, err := New(dir) + require.NoError(t, err) + defer db.Close() + + r, err := db.GetRawBody(175881) + require.NoError(t, err) + var body *types.Body + err = rlp.DecodeBytes(r, &body) + require.NoError(t, err) + require.NotNil(t, body, "block body not found") + assert.Equal(t, 3, len(body.Transactions)) + + r, err = db.GetRawReceipts(175881) + require.NoError(t, err) + var receipts []*types.ReceiptForStorage + err = rlp.DecodeBytes(r, &receipts) + require.NoError(t, err) + require.NotNil(t, receipts, "receipts not found") + assert.Equal(t, 3, len(receipts), "receipts length mismatch") + + // Cross-check against the era1 store: both backends must return the same + // storage encoding. + eraDB, err := New("testdata") + require.NoError(t, err) + defer eraDB.Close() + for _, num := range []uint64{0, 1024, 172032, 175881, 180223} { + want, err := eraDB.GetRawReceipts(num) + require.NoError(t, err) + got, err := db.GetRawReceipts(num) + require.NoError(t, err) + assert.Equal(t, want, got, "receipts mismatch at block %d", num) + + wantBody, err := eraDB.GetRawBody(num) + require.NoError(t, err) + gotBody, err := db.GetRawBody(num) + require.NoError(t, err) + assert.Equal(t, wantBody, gotBody, "body mismatch at block %d", num) + } +} + +// TestEreDatabaseNoReceipts checks that ere files written with the +// "noreceipts" profile are rejected by the store. The testdata fixture is a +// minimal single-block ere file whose index has no receipts component. +func TestEreDatabaseNoReceipts(t *testing.T) { + db, err := New("testdata/noreceipts") + require.NoError(t, err) + defer db.Close() + + _, err = db.GetRawBody(0) + require.ErrorContains(t, err, "no receipts") + _, err = db.GetRawReceipts(0) + require.ErrorContains(t, err, "no receipts") +} + +// convertEra1ToEre reads an era1 file and writes its contents as an ere file +// into dir, using the canonical ere file name. +func convertEra1ToEre(t *testing.T, era1Path, dir, network string, epoch int) { + t.Helper() + + e, err := onedb.Open(era1Path) + require.NoError(t, err) + defer e.Close() + + f, err := os.CreateTemp(dir, "ere-convert-*") + require.NoError(t, err) + defer f.Close() + + builder := execdb.NewBuilder(f) + td, err := e.InitialTD() + require.NoError(t, err) + + it, err := e.Iterator() + require.NoError(t, err) + for it.Next() { + block, receipts, err := it.BlockAndReceipts() + require.NoError(t, err) + td.Add(td, block.Difficulty()) + require.NoError(t, builder.Add(block, receipts, new(big.Int).Set(td))) + } + require.NoError(t, it.Error()) + + lastHash, err := builder.Finalize() + require.NoError(t, err) + require.NoError(t, f.Close()) + require.NoError(t, os.Rename(f.Name(), filepath.Join(dir, execdb.Filename(network, epoch, lastHash)))) +} + func TestEraDatabaseConcurrentOpen(t *testing.T) { db, err := New("testdata") require.NoError(t, err) diff --git a/core/rawdb/eradb/testdata/noreceipts/sepolia-00000-deadbeef-noreceipts.ere b/core/rawdb/eradb/testdata/noreceipts/sepolia-00000-deadbeef-noreceipts.ere new file mode 100644 index 0000000000000000000000000000000000000000..ab791d1e00858578a63c1f61d3fcc9a27e858cfa GIT binary patch literal 222 zcmYc_VgLhXhCnd=pN)Z`*e@}lAdA0rfPVq{=pY+z(;@@7=%fEdfbz``JkW-uc=1H%s^ojZpP9AINm f0828$RO6u2jSNs^lKw-$ZYX^NN;5-M!OQ~y*qbyH literal 0 HcmV?d00001 diff --git a/internal/era/execdb/reader.go b/internal/era/execdb/reader.go index e9831f9655..0133ca7ae2 100644 --- a/internal/era/execdb/reader.go +++ b/internal/era/execdb/reader.go @@ -204,6 +204,12 @@ func (e *Era) HasComponent(c componentType) bool { return ok } +// HasReceipts reports whether the file contains a receipts component. Files +// written with the "noreceipts" profile omit it. +func (e *Era) HasReceipts() bool { + return e.HasComponent(receipts) +} + // InitialTD returns initial total difficulty before the difficulty of the // first block of the Era is applied. Returns an error if TD is not available // (e.g., post-merge epoch). From 633ae4791e047744313ca164b798e304ac10921d Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi Date: Fri, 12 Jun 2026 11:18:51 +0000 Subject: [PATCH 2/2] update keeper go.mod --- cmd/keeper/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index b2caee8b63..e51dac185e 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -27,6 +27,7 @@ require ( github.com/golang/snappy v1.0.0 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.3.2 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect