Merge branch 'fix/era-store' of https://github.com/0xjvn/go-ethereum into fix/era-query

core/rawdb/eradb/eradb.go

@@ -56,6 +56,7 @@ type fileCacheEntry struct {
 	refcount int           // reference count. This is protected by Store.mu!
 	opened   chan struct{} // signals opening of file has completed
 	file     era.Era       // the file (era1 or ere)
+	slim     bool          // true if receipts are stored in the ere slim encoding
 	err      error         // error from opening the file
 }
 
@@ -150,12 +151,14 @@ func (db *Store) GetRawReceipts(number uint64) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return convertReceipts(data)
+	return convertReceipts(data, entry.slim)
 }
 
-// convertReceipts transforms an encoded block receipts list from the format
+// convertReceipts transforms an encoded block receipts list into the 'storage'
-// used by era1 into the 'storage' format used by the go-ethereum ancients database.
+// format used by the go-ethereum ancients database, i.e. a list of
-func convertReceipts(input []byte) ([]byte, error) {
+// [status, gas-used, logs]. The input uses the era1 network encoding, or the
+// ere slim encoding when slim is true.
+func convertReceipts(input []byte, slim bool) ([]byte, error) {
 	var (
 		out bytes.Buffer
 		enc = rlp.NewEncoderBuffer(&out)
@@ -166,11 +169,21 @@ func convertReceipts(input []byte) ([]byte, error) {
 	}
 	outerList := enc.List()
 	for i := 0; blockListIter.Next(); i++ {
+		var (
+			receiptData []byte
+			skip        int
+		)
+		if slim {
+			// Slim receipt is [tx-type, status, gas-used, logs]: skip the tx-type.
+			receiptData = blockListIter.Value()
+			skip = 0
+		} else {
+			// Era1 receipt is [status, gas-used, bloom, logs], prefixed by the
+			// tx type if non-legacy: skip the bloom.
 			kind, content, _, err := rlp.Split(blockListIter.Value())
 			if err != nil {
 				return nil, fmt.Errorf("receipt %d invalid: %v", i, err)
 			}
-		var receiptData []byte
 			switch kind {
 			case rlp.Byte:
 				return nil, fmt.Errorf("receipt %d is single byte", i)
@@ -181,17 +194,17 @@ func convertReceipts(input []byte) ([]byte, error) {
 				// Legacy receipt
 				receiptData = blockListIter.Value()
 			}
-		// Convert data list.
+			skip = 2
-		// Input is  [status, gas-used, bloom, logs]
+		}
-		// Output is [status, gas-used, logs], i.e. we need to skip the bloom.
+
 		dataIter, err := rlp.NewListIterator(receiptData)
 		if err != nil {
 			return nil, fmt.Errorf("receipt %d has invalid data: %v", i, err)
 		}
 		innerList := enc.List()
 		for field := 0; dataIter.Next(); field++ {
-			if field == 2 {
+			if field == skip {
-				continue // skip bloom
+				continue
 			}
 			enc.Write(dataIter.Value())
 		}
@@ -220,11 +233,11 @@ func (db *Store) getEraByEpoch(epoch uint64) *fileCacheEntry {
 
 	case fileIsNew:
 		// Open the file and put it into the cache.
-		e, err := db.openEraFile(epoch)
+		e, slim, err := db.openEraFile(epoch)
 		if err != nil {
 			db.fileFailedToOpen(epoch, entry, err)
 		} else {
-			db.fileOpened(epoch, entry, e)
+			db.fileOpened(epoch, entry, e, slim)
 		}
 		close(entry.opened)
 
@@ -268,7 +281,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC
 }
 
 // fileOpened is called after an era file has been successfully opened.
-func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file era.Era) {
+func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file era.Era, slim bool) {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
@@ -285,6 +298,7 @@ func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file era.Era) {
 
 	// Add it to the LRU. This may evict an existing item, which we have to close.
 	entry.file = file
+	entry.slim = slim
 	evictedEpoch, evictedEntry, _ := db.lru.Add3(epoch, entry)
 	if evictedEntry != nil {
 		evictedEntry.derefAndClose(evictedEpoch)
@@ -301,23 +315,24 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error
 	entry.err = err
 }
 
-func (db *Store) openEraFile(epoch uint64) (era.Era, error) {
+func (db *Store) openEraFile(epoch uint64) (era.Era, bool, error) {
 	// File name scheme is <network>-<epoch>-<root>.<ext>
 	// Try era1 first, then ere.
 	for _, ext := range []string{"era1", "ere"} {
 		glob := fmt.Sprintf("*-%05d-*.%s", epoch, ext)
 		matches, err := filepath.Glob(filepath.Join(db.datadir, glob))
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 		if len(matches) > 1 {
-			return nil, fmt.Errorf("multiple %s files found for epoch %d", ext, epoch)
+			return nil, false, fmt.Errorf("multiple %s files found for epoch %d", ext, epoch)
 		}
 		if len(matches) == 0 {
 			continue
 		}
 		filename := matches[0]
 		var e era.Era
+		slim := ext == "ere"
 		switch ext {
 		case "era1":
 			e, err = onedb.Open(filename)
@@ -325,22 +340,22 @@ func (db *Store) openEraFile(epoch uint64) (era.Era, error) {
 			// The era store serves receipts via RPC. Reject noreceipts
 			// profiles to avoid silently returning empty receipt data.
 			if strings.Contains(filepath.Base(filename), "-noreceipts") {
-				return nil, fmt.Errorf("era store does not support noreceipts profile: %s", filepath.Base(filename))
+				return nil, false, fmt.Errorf("era store does not support noreceipts profile: %s", filepath.Base(filename))
 			}
 			e, err = execdb.Open(filename)
 		}
 		if err != nil {
-			return nil, err
+			return nil, false, err
 		}
 		// Sanity-check start block.
 		if e.Start()%uint64(era.MaxSize) != 0 {
 			e.Close()
-			return nil, fmt.Errorf("%s file has invalid boundary. %d %% %d != 0", ext, e.Start(), era.MaxSize)
+			return nil, false, fmt.Errorf("%s file has invalid boundary. %d %% %d != 0", ext, e.Start(), era.MaxSize)
 		}
 		log.Debug("Opened era file", "type", ext, "epoch", epoch)
-		return e, nil
+		return e, slim, nil
 	}
-	return nil, fs.ErrNotExist
+	return nil, false, fs.ErrNotExist
 }
 
 // doneWithFile signals that the caller has finished using a file.

core/rawdb/eradb/eradb_test.go

@@ -17,12 +17,15 @@
 package eradb
 
 import (
+	"math/big"
 	"os"
 	"path/filepath"
 	"sync"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/internal/era/execdb"
+	"github.com/ethereum/go-ethereum/internal/era/onedb"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -50,6 +53,54 @@ func TestEraDatabase(t *testing.T) {
 	assert.Equal(t, 3, len(receipts), "receipts length mismatch")
 }
 
+// TestEreDatabase checks that the store can serve bodies and receipts from a
+// directory of ere files, and that the receipts returned are byte-identical to
+// the ones derived from the equivalent era1 files.
+func TestEreDatabase(t *testing.T) {
+	dir := t.TempDir()
+	convertEra1ToEre(t, "testdata/sepolia-00000-643a00f7.era1", dir, "sepolia", 0)
+	convertEra1ToEre(t, "testdata/sepolia-00021-b8814b14.era1", dir, "sepolia", 21)
+
+	db, err := New(dir)
+	require.NoError(t, err)
+	defer db.Close()
+
+	r, err := db.GetRawBody(175881)
+	require.NoError(t, err)
+	var body *types.Body
+	err = rlp.DecodeBytes(r, &body)
+	require.NoError(t, err)
+	require.NotNil(t, body, "block body not found")
+	assert.Equal(t, 3, len(body.Transactions))
+
+	r, err = db.GetRawReceipts(175881)
+	require.NoError(t, err)
+	var receipts []*types.ReceiptForStorage
+	err = rlp.DecodeBytes(r, &receipts)
+	require.NoError(t, err)
+	require.NotNil(t, receipts, "receipts not found")
+	assert.Equal(t, 3, len(receipts), "receipts length mismatch")
+
+	// Cross-check against the era1 store: both backends must return the same
+	// storage encoding.
+	eraDB, err := New("testdata")
+	require.NoError(t, err)
+	defer eraDB.Close()
+	for _, num := range []uint64{0, 1024, 172032, 175881, 180223} {
+		want, err := eraDB.GetRawReceipts(num)
+		require.NoError(t, err)
+		got, err := db.GetRawReceipts(num)
+		require.NoError(t, err)
+		assert.Equal(t, want, got, "receipts mismatch at block %d", num)
+
+		wantBody, err := eraDB.GetRawBody(num)
+		require.NoError(t, err)
+		gotBody, err := db.GetRawBody(num)
+		require.NoError(t, err)
+		assert.Equal(t, wantBody, gotBody, "body mismatch at block %d", num)
+	}
+}
+
 func TestEraStoreRejectsNoReceiptsProfile(t *testing.T) {
 	dir := t.TempDir()
 	stubName := "mainnet-00000-deadbeef-noreceipts.ere"
@@ -76,6 +127,39 @@ func TestEraStoreRejectsNoReceiptsProfile(t *testing.T) {
 	assert.Contains(t, err.Error(), "era store does not support noreceipts profile")
 }
 
+// convertEra1ToEre reads an era1 file and writes its contents as an ere file
+// into dir, using the canonical ere file name.
+func convertEra1ToEre(t *testing.T, era1Path, dir, network string, epoch int) {
+	t.Helper()
+
+	e, err := onedb.Open(era1Path)
+	require.NoError(t, err)
+	defer e.Close()
+
+	f, err := os.CreateTemp(dir, "ere-convert-*")
+	require.NoError(t, err)
+	defer f.Close()
+
+	builder := execdb.NewBuilder(f)
+	td, err := e.InitialTD()
+	require.NoError(t, err)
+
+	it, err := e.Iterator()
+	require.NoError(t, err)
+	for it.Next() {
+		block, receipts, err := it.BlockAndReceipts()
+		require.NoError(t, err)
+		td.Add(td, block.Difficulty())
+		require.NoError(t, builder.Add(block, receipts, new(big.Int).Set(td)))
+	}
+	require.NoError(t, it.Error())
+
+	lastHash, err := builder.Finalize()
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+	require.NoError(t, os.Rename(f.Name(), filepath.Join(dir, execdb.Filename(network, epoch, lastHash))))
+}
+
 func TestEraDatabaseConcurrentOpen(t *testing.T) {
 	db, err := New("testdata")
 	require.NoError(t, err)