Merge 5c510f4e4e into 7c9032dff6

cmd/keeper/go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.3.2 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect

core/rawdb/chain_freezer.go

@@ -447,3 +447,7 @@ func (f *chainFreezer) TruncateTail(group string, items uint64) (uint64, error)
 func (f *chainFreezer) SyncAncient() error {
 	return f.ancients.SyncAncient()
 }
+
+func (f *chainFreezer) EraStore() *eradb.Store {
+	return f.eradb
+}

core/rawdb/database.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb/eradb"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
@@ -86,6 +87,27 @@ func (frdb *freezerdb) Freeze() error {
 	return nil
 }
 
+// Unwrapper allows retrieving the underlying database from a wrapper.
+type Unwrapper interface {
+	Unwrap() ethdb.Database
+}
+
+// EraStore returns the eradb.Store from the database if available.
+// Returns nil if the database does not have an era store (e.g. in-memory or no freezer).
+func EraStore(db ethdb.Database) *eradb.Store {
+	for {
+		if frdb, ok := db.(*freezerdb); ok {
+			return frdb.EraStore()
+		}
+		if unwrapper, ok := db.(Unwrapper); ok {
+			db = unwrapper.Unwrap()
+		} else {
+			break
+		}
+	}
+	return nil
+}
+
 // nofreezedb is a database wrapper that disables freezer data retrievals.
 type nofreezedb struct {
 	ethdb.KeyValueStore

core/rawdb/eradb/eradb.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// Package eradb implements a history backend using era1 files.
+// Package eradb implements a history backend using era1 and ere files.
 package eradb
 
 import (
@@ -23,10 +23,13 @@ import (
 	"fmt"
 	"io/fs"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/era"
+	"github.com/ethereum/go-ethereum/internal/era/execdb"
 	"github.com/ethereum/go-ethereum/internal/era/onedb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
@@ -36,7 +39,7 @@ const openFileLimit = 64
 
 var errClosed = errors.New("era store is closed")
 
-// Store manages read access to a directory of era1 files.
+// Store manages read access to a directory of era1 and ere files.
 // The getter methods are thread-safe.
 type Store struct {
 	datadir string
@@ -52,7 +55,8 @@ type Store struct {
 type fileCacheEntry struct {
 	refcount int           // reference count. This is protected by Store.mu!
 	opened   chan struct{} // signals opening of file has completed
-	file     *onedb.Era    // the file
+	file     era.Era       // the file (era1 or ere)
+	slim     bool          // true if receipts are stored in the ere slim encoding
 	err      error         // error from opening the file
 }
 
@@ -101,6 +105,21 @@ func (db *Store) Close() {
 	}
 }
 
+// GetBlockByNumber returns the block for a given block number.
+func (db *Store) GetBlockByNumber(number uint64) (*types.Block, error) {
+	epoch := number / uint64(era.MaxSize)
+	entry := db.getEraByEpoch(epoch)
+	if entry.err != nil {
+		if errors.Is(entry.err, fs.ErrNotExist) {
+			return nil, nil
+		}
+		return nil, entry.err
+	}
+	defer db.doneWithFile(epoch, entry)
+
+	return entry.file.GetBlockByNumber(number)
+}
+
 // GetRawBody returns the raw body for a given block number.
 func (db *Store) GetRawBody(number uint64) ([]byte, error) {
 	epoch := number / uint64(era.MaxSize)
@@ -132,12 +151,14 @@ func (db *Store) GetRawReceipts(number uint64) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return convertReceipts(data)
+	return convertReceipts(data, entry.slim)
 }
 
-// convertReceipts transforms an encoded block receipts list from the format
+// convertReceipts transforms an encoded block receipts list into the 'storage'
-// used by era1 into the 'storage' format used by the go-ethereum ancients database.
+// format used by the go-ethereum ancients database, i.e. a list of
-func convertReceipts(input []byte) ([]byte, error) {
+// [status, gas-used, logs]. The input uses the era1 network encoding, or the
+// ere slim encoding when slim is true.
+func convertReceipts(input []byte, slim bool) ([]byte, error) {
 	var (
 		out bytes.Buffer
 		enc = rlp.NewEncoderBuffer(&out)
@@ -148,32 +169,42 @@ func convertReceipts(input []byte) ([]byte, error) {
 	}
 	outerList := enc.List()
 	for i := 0; blockListIter.Next(); i++ {
-		kind, content, _, err := rlp.Split(blockListIter.Value())
+		var (
-		if err != nil {
+			receiptData []byte
-			return nil, fmt.Errorf("receipt %d invalid: %v", i, err)
+			skip        int
-		}
+		)
-		var receiptData []byte
+		if slim {
-		switch kind {
+			// Slim receipt is [tx-type, status, gas-used, logs]: skip the tx-type.
-		case rlp.Byte:
-			return nil, fmt.Errorf("receipt %d is single byte", i)
-		case rlp.String:
-			// Typed receipt - skip type.
-			receiptData = content[1:]
-		case rlp.List:
-			// Legacy receipt
 			receiptData = blockListIter.Value()
+			skip = 0
+		} else {
+			// Era1 receipt is [status, gas-used, bloom, logs], prefixed by the
+			// tx type if non-legacy: skip the bloom.
+			kind, content, _, err := rlp.Split(blockListIter.Value())
+			if err != nil {
+				return nil, fmt.Errorf("receipt %d invalid: %v", i, err)
+			}
+			switch kind {
+			case rlp.Byte:
+				return nil, fmt.Errorf("receipt %d is single byte", i)
+			case rlp.String:
+				// Typed receipt - skip type.
+				receiptData = content[1:]
+			case rlp.List:
+				// Legacy receipt
+				receiptData = blockListIter.Value()
+			}
+			skip = 2
 		}
-		// Convert data list.
+
-		// Input is  [status, gas-used, bloom, logs]
-		// Output is [status, gas-used, logs], i.e. we need to skip the bloom.
 		dataIter, err := rlp.NewListIterator(receiptData)
 		if err != nil {
 			return nil, fmt.Errorf("receipt %d has invalid data: %v", i, err)
 		}
 		innerList := enc.List()
 		for field := 0; dataIter.Next(); field++ {
-			if field == 2 {
+			if field == skip {
-				continue // skip bloom
+				continue
 			}
 			enc.Write(dataIter.Value())
 		}
@@ -202,11 +233,11 @@ func (db *Store) getEraByEpoch(epoch uint64) *fileCacheEntry {
 
 	case fileIsNew:
 		// Open the file and put it into the cache.
-		e, err := db.openEraFile(epoch)
+		e, slim, err := db.openEraFile(epoch)
 		if err != nil {
 			db.fileFailedToOpen(epoch, entry, err)
 		} else {
-			db.fileOpened(epoch, entry, e)
+			db.fileOpened(epoch, entry, e, slim)
 		}
 		close(entry.opened)
 
@@ -250,7 +281,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC
 }
 
 // fileOpened is called after an era file has been successfully opened.
-func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) {
+func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file era.Era, slim bool) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
@@ -267,6 +298,7 @@ func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era
 
 	// Add it to the LRU. This may evict an existing item, which we have to close.
 	entry.file = file
+	entry.slim = slim
 	evictedEpoch, evictedEntry, _ := db.lru.Add3(epoch, entry)
 	if evictedEntry != nil {
 		evictedEntry.derefAndClose(evictedEpoch)
@@ -283,32 +315,47 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error
 	entry.err = err
 }
 
-func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) {
+func (db *Store) openEraFile(epoch uint64) (era.Era, bool, error) {
-	// File name scheme is <network>-<epoch>-<root>.
+	// File name scheme is <network>-<epoch>-<root>.<ext>
-	glob := fmt.Sprintf("*-%05d-*.era1", epoch)
+	// Try era1 first, then ere.
-	matches, err := filepath.Glob(filepath.Join(db.datadir, glob))
+	for _, ext := range []string{"era1", "ere"} {
-	if err != nil {
+		glob := fmt.Sprintf("*-%05d-*.%s", epoch, ext)
-		return nil, err
+		matches, err := filepath.Glob(filepath.Join(db.datadir, glob))
+		if err != nil {
+			return nil, false, err
+		}
+		if len(matches) > 1 {
+			return nil, false, fmt.Errorf("multiple %s files found for epoch %d", ext, epoch)
+		}
+		if len(matches) == 0 {
+			continue
+		}
+		filename := matches[0]
+		var e era.Era
+		slim := ext == "ere"
+		switch ext {
+		case "era1":
+			e, err = onedb.Open(filename)
+		case "ere":
+			// The era store serves receipts via RPC. Reject noreceipts
+			// profiles to avoid silently returning empty receipt data.
+			if strings.Contains(filepath.Base(filename), "-noreceipts") {
+				return nil, false, fmt.Errorf("era store does not support noreceipts profile: %s", filepath.Base(filename))
+			}
+			e, err = execdb.Open(filename)
+		}
+		if err != nil {
+			return nil, false, err
+		}
+		// Sanity-check start block.
+		if e.Start()%uint64(era.MaxSize) != 0 {
+			e.Close()
+			return nil, false, fmt.Errorf("%s file has invalid boundary. %d %% %d != 0", ext, e.Start(), era.MaxSize)
+		}
+		log.Debug("Opened era file", "type", ext, "epoch", epoch)
+		return e, slim, nil
 	}
-	if len(matches) > 1 {
+	return nil, false, fs.ErrNotExist
-		return nil, fmt.Errorf("multiple era1 files found for epoch %d", epoch)
-	}
-	if len(matches) == 0 {
-		return nil, fs.ErrNotExist
-	}
-	filename := matches[0]
-
-	e, err := onedb.Open(filename)
-	if err != nil {
-		return nil, err
-	}
-	// Sanity-check start block.
-	if e.Start()%uint64(era.MaxSize) != 0 {
-		e.Close()
-		return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize)
-	}
-	log.Debug("Opened era1 file", "epoch", epoch)
-	return e.(*onedb.Era), nil
 }
 
 // doneWithFile signals that the caller has finished using a file.
@@ -339,9 +386,9 @@ func (entry *fileCacheEntry) derefAndClose(epoch uint64) (closed bool) {
 
 	closeErr := entry.file.Close()
 	if closeErr == nil {
-		log.Debug("Closed era1 file", "epoch", epoch)
+		log.Debug("Closed era file", "epoch", epoch)
 	} else {
-		log.Warn("Error closing era1 file", "epoch", epoch, "err", closeErr)
+		log.Warn("Error closing era file", "epoch", epoch, "err", closeErr)
 	}
 	return true
 }

core/rawdb/eradb/eradb_test.go

@@ -17,10 +17,15 @@
 package eradb
 
 import (
+	"math/big"
+	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/internal/era/execdb"
+	"github.com/ethereum/go-ethereum/internal/era/onedb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -48,6 +53,113 @@ func TestEraDatabase(t *testing.T) {
 	assert.Equal(t, 3, len(receipts), "receipts length mismatch")
 }
 
+// TestEreDatabase checks that the store can serve bodies and receipts from a
+// directory of ere files, and that the receipts returned are byte-identical to
+// the ones derived from the equivalent era1 files.
+func TestEreDatabase(t *testing.T) {
+	dir := t.TempDir()
+	convertEra1ToEre(t, "testdata/sepolia-00000-643a00f7.era1", dir, "sepolia", 0)
+	convertEra1ToEre(t, "testdata/sepolia-00021-b8814b14.era1", dir, "sepolia", 21)
+
+	db, err := New(dir)
+	require.NoError(t, err)
+	defer db.Close()
+
+	r, err := db.GetRawBody(175881)
+	require.NoError(t, err)
+	var body *types.Body
+	err = rlp.DecodeBytes(r, &body)
+	require.NoError(t, err)
+	require.NotNil(t, body, "block body not found")
+	assert.Equal(t, 3, len(body.Transactions))
+
+	r, err = db.GetRawReceipts(175881)
+	require.NoError(t, err)
+	var receipts []*types.ReceiptForStorage
+	err = rlp.DecodeBytes(r, &receipts)
+	require.NoError(t, err)
+	require.NotNil(t, receipts, "receipts not found")
+	assert.Equal(t, 3, len(receipts), "receipts length mismatch")
+
+	// Cross-check against the era1 store: both backends must return the same
+	// storage encoding.
+	eraDB, err := New("testdata")
+	require.NoError(t, err)
+	defer eraDB.Close()
+	for _, num := range []uint64{0, 1024, 172032, 175881, 180223} {
+		want, err := eraDB.GetRawReceipts(num)
+		require.NoError(t, err)
+		got, err := db.GetRawReceipts(num)
+		require.NoError(t, err)
+		assert.Equal(t, want, got, "receipts mismatch at block %d", num)
+
+		wantBody, err := eraDB.GetRawBody(num)
+		require.NoError(t, err)
+		gotBody, err := db.GetRawBody(num)
+		require.NoError(t, err)
+		assert.Equal(t, wantBody, gotBody, "body mismatch at block %d", num)
+	}
+}
+
+func TestEraStoreRejectsNoReceiptsProfile(t *testing.T) {
+	dir := t.TempDir()
+	stubName := "mainnet-00000-deadbeef-noreceipts.ere"
+	stubPath := filepath.Join(dir, stubName)
+
+	// Write a non-empty stub so the glob finds the file. Contents don't matter
+	// because the noreceipts check fires before execdb.Open is called.
+	err := os.WriteFile(stubPath, []byte("stub"), 0644)
+	require.NoError(t, err)
+
+	db, err := New(dir)
+	require.NoError(t, err)
+	defer db.Close()
+
+	// Any block in epoch 0 should trigger the same rejection.
+	const block = uint64(0)
+
+	_, err = db.GetRawBody(block)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "era store does not support noreceipts profile")
+
+	_, err = db.GetRawReceipts(block)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "era store does not support noreceipts profile")
+}
+
+// convertEra1ToEre reads an era1 file and writes its contents as an ere file
+// into dir, using the canonical ere file name.
+func convertEra1ToEre(t *testing.T, era1Path, dir, network string, epoch int) {
+	t.Helper()
+
+	e, err := onedb.Open(era1Path)
+	require.NoError(t, err)
+	defer e.Close()
+
+	f, err := os.CreateTemp(dir, "ere-convert-*")
+	require.NoError(t, err)
+	defer f.Close()
+
+	builder := execdb.NewBuilder(f)
+	td, err := e.InitialTD()
+	require.NoError(t, err)
+
+	it, err := e.Iterator()
+	require.NoError(t, err)
+	for it.Next() {
+		block, receipts, err := it.BlockAndReceipts()
+		require.NoError(t, err)
+		td.Add(td, block.Difficulty())
+		require.NoError(t, builder.Add(block, receipts, new(big.Int).Set(td)))
+	}
+	require.NoError(t, it.Error())
+
+	lastHash, err := builder.Finalize()
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	require.NoError(t, os.Rename(f.Name(), filepath.Join(dir, execdb.Filename(network, epoch, lastHash))))
+}
+
 func TestEraDatabaseConcurrentOpen(t *testing.T) {
 	db, err := New("testdata")
 	require.NoError(t, err)

eth/api_backend.go

@@ -158,10 +158,19 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe
 		bn = b.HistoryPruningCutoff()
 	}
 	block := b.eth.blockchain.GetBlockByNumber(bn)
-	if block == nil && bn < b.HistoryPruningCutoff() {
+	if block != nil {
+		return block, nil
+	}
+	// Block not found in the local database, try the ERA store as a fallback.
+	if eraStore := rawdb.EraStore(b.eth.chainDb); eraStore != nil {
+		if eraBlock, err := eraStore.GetBlockByNumber(bn); err == nil && eraBlock != nil {
+			return eraBlock, nil
+		}
+	}
+	if bn < b.HistoryPruningCutoff() {
 		return nil, &history.PrunedHistoryError{}
 	}
-	return block, nil
+	return nil, nil
 }
 
 func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) {

node/node.go

@@ -781,6 +781,11 @@ func (db *closeTrackingDB) Close() error {
 	return err
 }
 
+// Unwrap returns the underlying database.
+func (db *closeTrackingDB) Unwrap() ethdb.Database {
+	return db.Database
+}
+
 // wrapDatabase ensures the database will be auto-closed when Node is closed.
 func (n *Node) wrapDatabase(db ethdb.Database) ethdb.Database {
 	wrapper := &closeTrackingDB{db, n}