feat: follower node sync from DA (#1098)

* port changes from #1013

* port changes from #1068

* go.mod tidy

* fix compile error

* fix goimports

* fix log

* address review comments

* upgrade golang.org/x/net to 0.23.0

* bump version

* remove unused flag

* update da-codec commit

---------

Co-authored-by: Péter Garamvölgyi <peter@scroll.io>

cmd/geth/main.go

@@ -171,6 +171,10 @@ var (
 		utils.CircuitCapacityCheckWorkersFlag,
 		utils.RollupVerifyEnabledFlag,
 		utils.ShadowforkPeersFlag,
+		utils.DASyncEnabledFlag,
+		utils.DABlockNativeAPIEndpointFlag,
+		utils.DABlobScanAPIEndpointFlag,
+		utils.DABeaconNodeAPIEndpointFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/utils/flags.go

@@ -875,6 +875,24 @@ var (
 		Name:  "net.shadowforkpeers",
 		Usage: "peer ids of shadow fork peers",
 	}
+
+	// DA syncing settings
+	DASyncEnabledFlag = &cli.BoolFlag{
+		Name:  "da.sync",
+		Usage: "Enable node syncing from DA",
+	}
+	DABlobScanAPIEndpointFlag = &cli.StringFlag{
+		Name:  "da.blob.blobscan",
+		Usage: "BlobScan blob API endpoint",
+	}
+	DABlockNativeAPIEndpointFlag = &cli.StringFlag{
+		Name:  "da.blob.blocknative",
+		Usage: "BlockNative blob API endpoint",
+	}
+	DABeaconNodeAPIEndpointFlag = &cli.StringFlag{
+		Name:  "da.blob.beaconnode",
+		Usage: "Beacon node API endpoint",
+	}
 )
 
 // MakeDataDir retrieves the currently requested data directory, terminating
@@ -1319,6 +1337,10 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	setSmartCard(ctx, cfg)
 	setL1(ctx, cfg)
 
+	if ctx.IsSet(DASyncEnabledFlag.Name) {
+		cfg.DaSyncingEnabled = ctx.Bool(DASyncEnabledFlag.Name)
+	}
+
 	if ctx.GlobalIsSet(ExternalSignerFlag.Name) {
 		cfg.ExternalSigner = ctx.GlobalString(ExternalSignerFlag.Name)
 	}
@@ -1604,6 +1626,21 @@ func setEnableRollupVerify(ctx *cli.Context, cfg *ethconfig.Config) {
 	}
 }
 
+func setDA(ctx *cli.Context, cfg *ethconfig.Config) {
+	if ctx.IsSet(DASyncEnabledFlag.Name) {
+		cfg.EnableDASyncing = ctx.Bool(DASyncEnabledFlag.Name)
+		if ctx.IsSet(DABlobScanAPIEndpointFlag.Name) {
+			cfg.DA.BlobScanAPIEndpoint = ctx.String(DABlobScanAPIEndpointFlag.Name)
+		}
+		if ctx.IsSet(DABlockNativeAPIEndpointFlag.Name) {
+			cfg.DA.BlockNativeAPIEndpoint = ctx.String(DABlockNativeAPIEndpointFlag.Name)
+		}
+		if ctx.IsSet(DABeaconNodeAPIEndpointFlag.Name) {
+			cfg.DA.BeaconNodeAPIEndpoint = ctx.String(DABeaconNodeAPIEndpointFlag.Name)
+		}
+	}
+}
+
 func setMaxBlockRange(ctx *cli.Context, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(MaxBlockRangeFlag.Name) {
 		cfg.MaxBlockRange = ctx.GlobalInt64(MaxBlockRangeFlag.Name)
@@ -1679,6 +1716,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setLes(ctx, cfg)
 	setCircuitCapacityCheck(ctx, cfg)
 	setEnableRollupVerify(ctx, cfg)
+	setDA(ctx, cfg)
 	setMaxBlockRange(ctx, cfg)
 	if ctx.GlobalIsSet(ShadowforkPeersFlag.Name) {
 		cfg.ShadowForkPeerIDs = ctx.GlobalStringSlice(ShadowforkPeersFlag.Name)

common/backoff/exponential.go

@@ -0,0 +1,51 @@
+package backoff
+
+import (
+	"math"
+	"math/rand"
+	"time"
+)
+
+// Exponential is a backoff strategy that increases the delay between retries exponentially.
+type Exponential struct {
+	attempt int
+
+	maxJitter time.Duration
+
+	min time.Duration
+	max time.Duration
+}
+
+func NewExponential(minimum, maximum, maxJitter time.Duration) *Exponential {
+	return &Exponential{
+		min:       minimum,
+		max:       maximum,
+		maxJitter: maxJitter,
+	}
+}
+
+func (e *Exponential) NextDuration() time.Duration {
+	var jitter time.Duration
+	if e.maxJitter > 0 {
+		jitter = time.Duration(rand.Int63n(e.maxJitter.Nanoseconds()))
+	}
+
+	minFloat := float64(e.min)
+	duration := math.Pow(2, float64(e.attempt)) * minFloat
+
+	// limit at configured maximum
+	if duration > float64(e.max) {
+		duration = float64(e.max)
+	}
+
+	e.attempt++
+	return time.Duration(duration) + jitter
+}
+
+func (e *Exponential) Reset() {
+	e.attempt = 0
+}
+
+func (e *Exponential) Attempt() int {
+	return e.attempt
+}

common/backoff/exponential_test.go

@@ -0,0 +1,39 @@
+package backoff
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExponentialBackoff(t *testing.T) {
+	t.Run("Multiple attempts", func(t *testing.T) {
+		e := NewExponential(100*time.Millisecond, 10*time.Second, 0)
+		expectedDurations := []time.Duration{
+			100 * time.Millisecond,
+			200 * time.Millisecond,
+			400 * time.Millisecond,
+			800 * time.Millisecond,
+			1600 * time.Millisecond,
+			3200 * time.Millisecond,
+			6400 * time.Millisecond,
+			10 * time.Second, // capped at max
+		}
+		for i, expected := range expectedDurations {
+			require.Equal(t, expected, e.NextDuration(), "attempt %d", i)
+		}
+	})
+
+	t.Run("Jitter added", func(t *testing.T) {
+		e := NewExponential(1*time.Second, 10*time.Second, 1*time.Second)
+		duration := e.NextDuration()
+		require.GreaterOrEqual(t, duration, 1*time.Second)
+		require.Less(t, duration, 2*time.Second)
+	})
+
+	t.Run("Edge case: min > max", func(t *testing.T) {
+		e := NewExponential(10*time.Second, 5*time.Second, 0)
+		require.Equal(t, 5*time.Second, e.NextDuration())
+	})
+}

common/heap.go

@@ -0,0 +1,109 @@
+package common
+
+import (
+	"container/heap"
+)
+
+// Heap is a generic min-heap (or max-heap, depending on Comparable behavior) implementation.
+type Heap[T Comparable[T]] struct {
+	heap innerHeap[T]
+}
+
+func NewHeap[T Comparable[T]]() *Heap[T] {
+	return &Heap[T]{
+		heap: make(innerHeap[T], 0),
+	}
+}
+
+func (h *Heap[T]) Len() int {
+	return len(h.heap)
+}
+
+func (h *Heap[T]) Push(element T) *HeapElement[T] {
+	heapElement := NewHeapElement(element)
+	heap.Push(&h.heap, heapElement)
+
+	return heapElement
+}
+
+func (h *Heap[T]) Pop() *HeapElement[T] {
+	return heap.Pop(&h.heap).(*HeapElement[T])
+}
+
+func (h *Heap[T]) Peek() *HeapElement[T] {
+	if h.Len() == 0 {
+		return nil
+	}
+
+	return h.heap[0]
+}
+
+func (h *Heap[T]) Remove(element *HeapElement[T]) {
+	heap.Remove(&h.heap, element.index)
+}
+
+func (h *Heap[T]) Clear() {
+	h.heap = make(innerHeap[T], 0)
+}
+
+type innerHeap[T Comparable[T]] []*HeapElement[T]
+
+func (h innerHeap[T]) Len() int {
+	return len(h)
+}
+
+func (h innerHeap[T]) Less(i, j int) bool {
+	return h[i].Value().CompareTo(h[j].Value()) < 0
+}
+
+func (h innerHeap[T]) Swap(i, j int) {
+	h[i], h[j] = h[j], h[i]
+	h[i].index, h[j].index = i, j
+}
+
+func (h *innerHeap[T]) Push(x interface{}) {
+	data := x.(*HeapElement[T])
+	*h = append(*h, data)
+	data.index = len(*h) - 1
+}
+
+func (h *innerHeap[T]) Pop() interface{} {
+	n := len(*h)
+	element := (*h)[n-1]
+	(*h)[n-1] = nil // avoid memory leak
+	*h = (*h)[:n-1]
+	element.index = -1
+
+	return element
+}
+
+// Comparable is an interface for types that can be compared.
+type Comparable[T any] interface {
+	// CompareTo compares x with other.
+	// To create a min heap, return:
+	//	-1 if x <  other
+	//	 0 if x == other
+	//	+1 if x >  other
+	// To create a max heap, return the opposite.
+	CompareTo(other T) int
+}
+
+// HeapElement is a wrapper around the value stored in the heap.
+type HeapElement[T Comparable[T]] struct {
+	value T
+	index int
+}
+
+func NewHeapElement[T Comparable[T]](value T) *HeapElement[T] {
+	return &HeapElement[T]{
+		value: value,
+	}
+}
+
+func (h *HeapElement[T]) Value() T {
+	return h.value
+}
+
+func (h *HeapElement[T]) Index() int {
+	return h.index
+}

common/heap_test.go

@@ -0,0 +1,40 @@
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type Int int
+
+func (i Int) CompareTo(other Int) int {
+	if i < other {
+		return -1
+	} else if i > other {
+		return 1
+	} else {
+		return 0
+	}
+}
+
+func TestHeap(t *testing.T) {
+	h := NewHeap[Int]()
+
+	require.Equal(t, 0, h.Len(), "Heap should be empty initially")
+
+	h.Push(Int(3))
+	h.Push(Int(1))
+	h.Push(Int(2))
+
+	require.Equal(t, 3, h.Len(), "Heap should have three elements after pushing")
+
+	require.EqualValues(t, 1, h.Pop(), "Pop should return the smallest element")
+	require.Equal(t, 2, h.Len(), "Heap should have two elements after popping")
+
+	require.EqualValues(t, 2, h.Pop(), "Pop should return the next smallest element")
+	require.Equal(t, 1, h.Len(), "Heap should have one element after popping")
+
+	require.EqualValues(t, 3, h.Pop(), "Pop should return the last element")
+	require.Equal(t, 0, h.Len(), "Heap should be empty after popping all elements")
+}

common/shrinkingmap.go

@@ -0,0 +1,71 @@
+package common
+
+// ShrinkingMap is a map that shrinks itself (by allocating a new map) after a certain number of deletions have been performed.
+// If shrinkAfterDeletionsCount is set to <=0, the map will never shrink.
+// This is useful to prevent memory leaks in long-running processes that delete a lot of keys from a map.
+// See here for more details: https://github.com/golang/go/issues/20135
+type ShrinkingMap[K comparable, V any] struct {
+	m           map[K]V
+	deletedKeys int
+
+	shrinkAfterDeletionsCount int
+}
+
+func NewShrinkingMap[K comparable, V any](shrinkAfterDeletionsCount int) *ShrinkingMap[K, V] {
+	return &ShrinkingMap[K, V]{
+		m:                         make(map[K]V),
+		shrinkAfterDeletionsCount: shrinkAfterDeletionsCount,
+	}
+}
+
+func (s *ShrinkingMap[K, V]) Set(key K, value V) {
+	s.m[key] = value
+}
+
+func (s *ShrinkingMap[K, V]) Get(key K) (value V, exists bool) {
+	value, exists = s.m[key]
+	return value, exists
+}
+
+func (s *ShrinkingMap[K, V]) Has(key K) bool {
+	_, exists := s.m[key]
+	return exists
+}
+
+func (s *ShrinkingMap[K, V]) Delete(key K) (deleted bool) {
+	if _, exists := s.m[key]; !exists {
+		return false
+	}
+
+	delete(s.m, key)
+	s.deletedKeys++
+
+	if s.shouldShrink() {
+		s.shrink()
+	}
+
+	return true
+}
+
+func (s *ShrinkingMap[K, V]) Size() (size int) {
+	return len(s.m)
+}
+
+func (s *ShrinkingMap[K, V]) Clear() {
+	s.m = make(map[K]V)
+	s.deletedKeys = 0
+}
+
+func (s *ShrinkingMap[K, V]) shouldShrink() bool {
+	return s.shrinkAfterDeletionsCount > 0 && s.deletedKeys >= s.shrinkAfterDeletionsCount
+}
+
+func (s *ShrinkingMap[K, V]) shrink() {
+	newMap := make(map[K]V, len(s.m))
+	for k, v := range s.m {
+		newMap[k] = v
+	}
+
+	s.m = newMap
+	s.deletedKeys = 0
+}

common/shrinkingmap_test.go

@@ -0,0 +1,135 @@
+package common
+
+import (
+	"fmt"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestShrinkingMap_Shrink(t *testing.T) {
+	m := NewShrinkingMap[int, int](10)
+
+	for i := 0; i < 100; i++ {
+		m.Set(i, i)
+	}
+
+	for i := 0; i < 100; i++ {
+		val, exists := m.Get(i)
+		require.Equal(t, true, exists)
+		require.Equal(t, i, val)
+
+		has := m.Has(i)
+		require.Equal(t, true, has)
+	}
+
+	for i := 0; i < 9; i++ {
+		m.Delete(i)
+	}
+	require.Equal(t, 9, m.deletedKeys)
+
+	// Delete the 10th key -> shrinks the map
+	m.Delete(9)
+	require.Equal(t, 0, m.deletedKeys)
+
+	for i := 0; i < 100; i++ {
+		if i < 10 {
+			val, exists := m.Get(i)
+			require.Equal(t, false, exists)
+			require.Equal(t, 0, val)
+
+			has := m.Has(i)
+			require.Equal(t, false, has)
+		} else {
+			val, exists := m.Get(i)
+			require.Equal(t, true, exists)
+			require.Equal(t, i, val)
+
+			has := m.Has(i)
+			require.Equal(t, true, has)
+		}
+	}
+
+	require.Equal(t, 90, m.Size())
+}
+
+func TestNewShrinkingMap_NoShrinking(t *testing.T) {
+	m := NewShrinkingMap[int, int](0)
+	for i := 0; i < 10000; i++ {
+		m.Set(i, i)
+	}
+
+	for i := 0; i < 10000; i++ {
+		val, exists := m.Get(i)
+		require.Equal(t, true, exists)
+		require.Equal(t, i, val)
+
+		m.Delete(i)
+	}
+
+	require.Equal(t, 0, m.Size())
+	require.Equal(t, 10000, m.deletedKeys)
+}
+
+func TestShrinkingMap_MemoryShrinking(t *testing.T) {
+	t.Skip("Only for manual testing and memory profiling")
+
+	gcAndPrintAlloc("start")
+	m := NewShrinkingMap[int, int](10000)
+
+	const mapSize = 1_000_000
+
+	for i := 0; i < mapSize; i++ {
+		m.Set(i, i)
+	}
+
+	gcAndPrintAlloc("after map creation")
+
+	for i := 0; i < mapSize/2; i++ {
+		m.Delete(i)
+	}
+
+	gcAndPrintAlloc("after removing half of the elements")
+
+	val, exist := m.Get(mapSize - 1)
+	require.Equal(t, true, exist)
+	require.Equal(t, mapSize-1, val)
+
+	gcAndPrintAlloc("end")
+}
+
+func TestShrinkingMap_MemoryNoShrinking(t *testing.T) {
+	t.Skip("Only for manual testing and memory profiling")
+
+	gcAndPrintAlloc("start")
+	m := NewShrinkingMap[int, int](0)
+
+	const mapSize = 1_000_000
+
+	for i := 0; i < mapSize; i++ {
+		m.Set(i, i)
+	}
+
+	gcAndPrintAlloc("after map creation")
+
+	for i := 0; i < mapSize/2; i++ {
+		m.Delete(i)
+	}
+
+	gcAndPrintAlloc("after removing half of the elements")
+
+	val, exist := m.Get(mapSize - 1)
+	require.Equal(t, true, exist)
+	require.Equal(t, mapSize-1, val)
+
+	gcAndPrintAlloc("end")
+}
+
+func gcAndPrintAlloc(prefix string) {
+	runtime.GC()
+
+	var stats runtime.MemStats
+	runtime.ReadMemStats(&stats)
+	fmt.Printf(prefix+", Allocated memory %d KiB\n", stats.Alloc/1024)
+}

core/blockchain.go

@@ -1803,6 +1803,56 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er
 	return it.index, err
 }
 
+func (bc *BlockChain) BuildAndWriteBlock(parentBlock *types.Block, header *types.Header, txs types.Transactions) (WriteStatus, error) {
+	if !bc.chainmu.TryLock() {
+		return NonStatTy, errInsertionInterrupted
+	}
+	defer bc.chainmu.Unlock()
+
+	statedb, err := state.New(parentBlock.Root(), bc.stateCache, bc.snaps)
+	if err != nil {
+		return NonStatTy, err
+	}
+
+	statedb.StartPrefetcher("l1sync")
+	defer statedb.StopPrefetcher()
+
+	header.ParentHash = parentBlock.Hash()
+
+	tempBlock := types.NewBlockWithHeader(header).WithBody(txs, nil)
+	receipts, logs, gasUsed, err := bc.processor.Process(tempBlock, statedb, bc.vmConfig)
+	if err != nil {
+		return NonStatTy, fmt.Errorf("error processing block: %w", err)
+	}
+
+	// TODO: once we have the extra and difficulty we need to verify the signature of the block with Clique
+	//  This should be done with https://github.com/scroll-tech/go-ethereum/pull/913.
+
+	// finalize and assemble block as fullBlock
+	header.GasUsed = gasUsed
+	header.Root = statedb.IntermediateRoot(bc.chainConfig.IsEIP158(header.Number))
+
+	fullBlock := types.NewBlock(header, txs, nil, receipts, trie.NewStackTrie(nil))
+
+	blockHash := fullBlock.Hash()
+	// manually replace the block hash in the receipts
+	for i, receipt := range receipts {
+		// add block location fields
+		receipt.BlockHash = blockHash
+		receipt.BlockNumber = tempBlock.Number()
+		receipt.TransactionIndex = uint(i)
+
+		for _, l := range receipt.Logs {
+			l.BlockHash = blockHash
+		}
+	}
+	for _, l := range logs {
+		l.BlockHash = blockHash
+	}
+
+	return bc.writeBlockWithState(fullBlock, receipts, logs, statedb, false)
+}
+
 // insertSideChain is called when an import batch hits upon a pruned ancestor
 // error, which happens when a sidechain with a sufficiently old fork-block is
 // found.

core/rawdb/accessors_da_syncer.go

@@ -0,0 +1,39 @@
+package rawdb
+
+import (
+	"math/big"
+
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/log"
+)
+
+// WriteDASyncedL1BlockNumber writes the highest synced L1 block number to the database.
+func WriteDASyncedL1BlockNumber(db ethdb.KeyValueWriter, L1BlockNumber uint64) {
+	value := big.NewInt(0).SetUint64(L1BlockNumber).Bytes()
+
+	if err := db.Put(daSyncedL1BlockNumberKey, value); err != nil {
+		log.Crit("Failed to update DA synced L1 block number", "err", err)
+	}
+}
+
+// ReadDASyncedL1BlockNumber retrieves the highest synced L1 block number.
+func ReadDASyncedL1BlockNumber(db ethdb.Reader) *uint64 {
+	data, err := db.Get(daSyncedL1BlockNumberKey)
+	if err != nil && isNotFoundErr(err) {
+		return nil
+	}
+	if err != nil {
+		log.Crit("Failed to read DA synced L1 block number from database", "err", err)
+	}
+	if len(data) == 0 {
+		return nil
+	}
+
+	number := new(big.Int).SetBytes(data)
+	if !number.IsUint64() {
+		log.Crit("Unexpected DA synced L1 block number in database", "number", number)
+	}
+
+	value := number.Uint64()
+	return &value
+}

core/rawdb/accessors_rollup_event.go

@@ -58,47 +58,6 @@ func ReadRollupEventSyncedL1BlockNumber(db ethdb.Reader) *uint64 {
 	return &rollupEventSyncedL1BlockNumber
 }
 
-// WriteBatchChunkRanges writes the block ranges for each chunk within a batch to the database.
-// It serializes the chunk ranges using RLP and stores them under a key derived from the batch index.
-// for backward compatibility, new info is also stored in CommittedBatchMeta.
-func WriteBatchChunkRanges(db ethdb.KeyValueWriter, batchIndex uint64, chunkBlockRanges []*ChunkBlockRange) {
-	value, err := rlp.EncodeToBytes(chunkBlockRanges)
-	if err != nil {
-		log.Crit("failed to RLP encode batch chunk ranges", "batch index", batchIndex, "err", err)
-	}
-	if err := db.Put(batchChunkRangesKey(batchIndex), value); err != nil {
-		log.Crit("failed to store batch chunk ranges", "batch index", batchIndex, "value", value, "err", err)
-	}
-}
-
-// DeleteBatchChunkRanges removes the block ranges of all chunks associated with a specific batch from the database.
-// Note: Only non-finalized batches can be reverted.
-// for backward compatibility, new info is also stored in CommittedBatchMeta.
-func DeleteBatchChunkRanges(db ethdb.KeyValueWriter, batchIndex uint64) {
-	if err := db.Delete(batchChunkRangesKey(batchIndex)); err != nil {
-		log.Crit("failed to delete batch chunk ranges", "batch index", batchIndex, "err", err)
-	}
-}
-
-// ReadBatchChunkRanges retrieves the block ranges of all chunks associated with a specific batch from the database.
-// It returns a list of ChunkBlockRange pointers, or nil if no chunk ranges are found for the given batch index.
-// for backward compatibility, new info is also stored in CommittedBatchMeta.
-func ReadBatchChunkRanges(db ethdb.Reader, batchIndex uint64) []*ChunkBlockRange {
-	data, err := db.Get(batchChunkRangesKey(batchIndex))
-	if err != nil && isNotFoundErr(err) {
-		return nil
-	}
-	if err != nil {
-		log.Crit("failed to read batch chunk ranges from database", "err", err)
-	}
-
-	cr := new([]*ChunkBlockRange)
-	if err := rlp.Decode(bytes.NewReader(data), cr); err != nil {
-		log.Crit("Invalid ChunkBlockRange RLP", "batch index", batchIndex, "data", data, "err", err)
-	}
-	return *cr
-}
-
 // WriteFinalizedBatchMeta stores the metadata of a finalized batch in the database.
 func WriteFinalizedBatchMeta(db ethdb.KeyValueWriter, batchIndex uint64, finalizedBatchMeta *FinalizedBatchMeta) {
 	value, err := rlp.EncodeToBytes(finalizedBatchMeta)

core/rawdb/accessors_rollup_event_test.go

@@ -147,70 +147,6 @@ func TestFinalizedBatchMeta(t *testing.T) {
 	}
 }
 
-func TestBatchChunkRanges(t *testing.T) {
-	chunks := [][]*ChunkBlockRange{
-		{
-			{StartBlockNumber: 1, EndBlockNumber: 100},
-			{StartBlockNumber: 101, EndBlockNumber: 200},
-		},
-		{
-			{StartBlockNumber: 201, EndBlockNumber: 300},
-			{StartBlockNumber: 301, EndBlockNumber: 400},
-		},
-		{
-			{StartBlockNumber: 401, EndBlockNumber: 500},
-		},
-	}
-
-	db := NewMemoryDatabase()
-
-	for i, chunkRange := range chunks {
-		batchIndex := uint64(i)
-		WriteBatchChunkRanges(db, batchIndex, chunkRange)
-	}
-
-	for i, chunkRange := range chunks {
-		batchIndex := uint64(i)
-		readChunkRange := ReadBatchChunkRanges(db, batchIndex)
-		if len(readChunkRange) != len(chunkRange) {
-			t.Fatal("Mismatch in number of chunk ranges", "expected", len(chunkRange), "got", len(readChunkRange))
-		}
-
-		for j, cr := range readChunkRange {
-			if cr.StartBlockNumber != chunkRange[j].StartBlockNumber || cr.EndBlockNumber != chunkRange[j].EndBlockNumber {
-				t.Fatal("Mismatch in chunk range", "batch index", batchIndex, "expected", chunkRange[j], "got", cr)
-			}
-		}
-	}
-
-	// over-write
-	newRange := []*ChunkBlockRange{{StartBlockNumber: 1001, EndBlockNumber: 1100}}
-	WriteBatchChunkRanges(db, 0, newRange)
-	readChunkRange := ReadBatchChunkRanges(db, 0)
-	if len(readChunkRange) != 1 || readChunkRange[0].StartBlockNumber != 1001 || readChunkRange[0].EndBlockNumber != 1100 {
-		t.Fatal("Over-write failed for chunk range", "expected", newRange, "got", readChunkRange)
-	}
-
-	// read non-existing value
-	if readChunkRange = ReadBatchChunkRanges(db, uint64(len(chunks)+1)); readChunkRange != nil {
-		t.Fatal("Expected nil for non-existing value", "got", readChunkRange)
-	}
-
-	// delete: revert batch
-	for i := range chunks {
-		batchIndex := uint64(i)
-		DeleteBatchChunkRanges(db, batchIndex)
-
-		readChunkRange := ReadBatchChunkRanges(db, batchIndex)
-		if readChunkRange != nil {
-			t.Fatal("Chunk range was not deleted", "batch index", batchIndex)
-		}
-	}
-
-	// delete non-existing value: ensure the delete operation handles non-existing values without errors.
-	DeleteBatchChunkRanges(db, uint64(len(chunks)+1))
-}
-
 func TestWriteReadDeleteCommittedBatchMeta(t *testing.T) {
 	db := NewMemoryDatabase()
 

core/rawdb/schema.go

@@ -112,7 +112,6 @@ var (
 
 	// Scroll rollup event store
 	rollupEventSyncedL1BlockNumberKey = []byte("R-LastRollupEventSyncedL1BlockNumber")
-	batchChunkRangesPrefix            = []byte("R-bcr")
 	batchMetaPrefix                   = []byte("R-bm")
 	finalizedL2BlockNumberKey         = []byte("R-finalized")
 	lastFinalizedBatchIndexKey        = []byte("R-finalizedBatchIndex")
@@ -125,6 +124,9 @@ var (
 	numSkippedTransactionsKey    = []byte("NumberOfSkippedTransactions")
 	skippedTransactionPrefix     = []byte("skip") // skippedTransactionPrefix + tx hash -> skipped transaction
 	skippedTransactionHashPrefix = []byte("sh")   // skippedTransactionHashPrefix + index -> tx hash
+
+	// Scroll da syncer store
+	daSyncedL1BlockNumberKey = []byte("LastDASyncedL1BlockNumber")
 )
 
 // Use the updated "L1" prefix on all new networks
@@ -301,11 +303,6 @@ func SkippedTransactionHashKey(index uint64) []byte {
 	return append(skippedTransactionHashPrefix, encodeBigEndian(index)...)
 }
 
-// batchChunkRangesKey = batchChunkRangesPrefix + batch index (uint64 big endian)
-func batchChunkRangesKey(batchIndex uint64) []byte {
-	return append(batchChunkRangesPrefix, encodeBigEndian(batchIndex)...)
-}
-
 // batchMetaKey = batchMetaPrefix + batch index (uint64 big endian)
 func batchMetaKey(batchIndex uint64) []byte {
 	return append(batchMetaPrefix, encodeBigEndian(batchIndex)...)

eth/backend.go

@@ -56,6 +56,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/params"
 	"github.com/scroll-tech/go-ethereum/rlp"
 	"github.com/scroll-tech/go-ethereum/rollup/ccc"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer"
 	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
 	"github.com/scroll-tech/go-ethereum/rollup/sync_service"
 	"github.com/scroll-tech/go-ethereum/rpc"
@@ -70,10 +71,12 @@ type Ethereum struct {
 	config *ethconfig.Config
 
 	// Handlers
-	txPool             *core.TxPool
-	syncService        *sync_service.SyncService
-	rollupSyncService  *rollup_sync_service.RollupSyncService
-	asyncChecker       *ccc.AsyncChecker
+	txPool            *core.TxPool
+	syncService       *sync_service.SyncService
+	rollupSyncService *rollup_sync_service.RollupSyncService
+	asyncChecker      *ccc.AsyncChecker
+	syncingPipeline   *da_syncer.SyncingPipeline
+
 	blockchain         *core.BlockChain
 	handler            *handler
 	ethDialCandidates  enode.Iterator
@@ -220,6 +223,18 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client sync_service.EthCl
 	}
 	eth.txPool = core.NewTxPool(config.TxPool, chainConfig, eth.blockchain)
 
+	// Initialize and start DA syncing pipeline before SyncService as SyncService is blocking until all L1 messages are loaded.
+	// We need SyncService to load the L1 messages for DA syncing, but since both sync from last known L1 state, we can
+	// simply let them run simultaneously. If messages are missing in DA syncing, it will be handled by the syncing pipeline
+	// by waiting and retrying.
+	if config.EnableDASyncing {
+		eth.syncingPipeline, err = da_syncer.NewSyncingPipeline(context.Background(), eth.blockchain, chainConfig, eth.chainDb, l1Client, stack.Config().L1DeploymentBlock, config.DA)
+		if err != nil {
+			return nil, fmt.Errorf("cannot initialize da syncer: %w", err)
+		}
+		eth.syncingPipeline.Start()
+	}
+
 	// initialize and start L1 message sync service
 	eth.syncService, err = sync_service.NewSyncService(context.Background(), chainConfig, stack.Config(), eth.chainDb, l1Client)
 	if err != nil {
@@ -257,7 +272,7 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client sync_service.EthCl
 		return nil, err
 	}
 
-	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock)
+	eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock, config.EnableDASyncing)
 	eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
 
 	eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
@@ -330,6 +345,15 @@ func (s *Ethereum) APIs() []rpc.API {
 	// Append any APIs exposed explicitly by the consensus engine
 	apis = append(apis, s.engine.APIs(s.BlockChain())...)
 
+	if !s.config.EnableDASyncing {
+		apis = append(apis, rpc.API{
+			Namespace: "eth",
+			Version:   "1.0",
+			Service:   downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux),
+			Public:    true,
+		})
+	}
+
 	// Append all the local APIs and return
 	return append(apis, []rpc.API{
 		{
@@ -342,11 +366,6 @@ func (s *Ethereum) APIs() []rpc.API {
 			Version:   "1.0",
 			Service:   NewPublicMinerAPI(s),
 			Public:    true,
-		}, {
-			Namespace: "eth",
-			Version:   "1.0",
-			Service:   downloader.NewPublicDownloaderAPI(s.handler.downloader, s.eventMux),
-			Public:    true,
 		}, {
 			Namespace: "miner",
 			Version:   "1.0",
@@ -553,6 +572,11 @@ func (s *Ethereum) SyncService() *sync_service.SyncService { return s.syncServic
 // Protocols returns all the currently configured
 // network protocols to start.
 func (s *Ethereum) Protocols() []p2p.Protocol {
+	// if DA syncing enabled then we don't create handler
+	if s.config.EnableDASyncing {
+		return nil
+	}
+
 	protos := eth.MakeProtocols((*ethHandler)(s.handler), s.networkID, s.ethDialCandidates)
 	if !s.blockchain.Config().Scroll.ZktrieEnabled() && s.config.SnapshotCache > 0 {
 		protos = append(protos, snap.MakeProtocols((*snapHandler)(s.handler), s.snapDialCandidates)...)
@@ -577,7 +601,11 @@ func (s *Ethereum) Start() error {
 	//	maxPeers -= s.config.LightPeers
 	//}
 	// Start the networking layer and the light server if requested
-	s.handler.Start(maxPeers)
+
+	// handler is not enabled when DA syncing enabled
+	if !s.config.EnableDASyncing {
+		s.handler.Start(maxPeers)
+	}
 	return nil
 }
 
@@ -587,7 +615,10 @@ func (s *Ethereum) Stop() error {
 	// Stop all the peer-related stuff first.
 	s.ethDialCandidates.Close()
 	s.snapDialCandidates.Close()
-	s.handler.Stop()
+	// handler is not enabled if DA syncing enabled
+	if !s.config.EnableDASyncing {
+		s.handler.Stop()
+	}
 
 	// Then stop everything else.
 	s.bloomIndexer.Close()
@@ -597,6 +628,9 @@ func (s *Ethereum) Stop() error {
 	if s.config.EnableRollupVerify {
 		s.rollupSyncService.Stop()
 	}
+	if s.config.EnableDASyncing {
+		s.syncingPipeline.Stop()
+	}
 	s.miner.Close()
 	if s.config.CheckCircuitCapacity {
 		s.asyncChecker.Wait()

eth/ethconfig/config.go

@@ -37,6 +37,7 @@ import (
 	"github.com/scroll-tech/go-ethereum/miner"
 	"github.com/scroll-tech/go-ethereum/node"
 	"github.com/scroll-tech/go-ethereum/params"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer"
 )
 
 // FullNodeGPO contains default gasprice oracle settings for full node.
@@ -218,6 +219,12 @@ type Config struct {
 
 	// List of peer ids that take part in the shadow-fork
 	ShadowForkPeerIDs []string
+
+	// Enable syncing node from DA
+	EnableDASyncing bool
+
+	// DA syncer options
+	DA da_syncer.Config
 }
 
 // CreateConsensusEngine creates a consensus engine for the given chain configuration.

go.mod

@@ -4,7 +4,7 @@ go 1.21
 
 require (
 	github.com/Azure/azure-storage-blob-go v0.7.0
-	github.com/VictoriaMetrics/fastcache v1.12.1
+	github.com/VictoriaMetrics/fastcache v1.12.2
 	github.com/aws/aws-sdk-go-v2 v1.2.0
 	github.com/aws/aws-sdk-go-v2/config v1.1.1
 	github.com/aws/aws-sdk-go-v2/credentials v1.1.1
@@ -50,7 +50,7 @@ require (
 	github.com/prometheus/tsdb v0.7.1
 	github.com/rjeczalik/notify v0.9.1
 	github.com/rs/cors v1.7.0
-	github.com/scroll-tech/da-codec v0.1.1-0.20240822151711-9e32313056ac
+	github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5
 	github.com/scroll-tech/zktrie v0.8.4
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sourcegraph/conc v0.3.0
@@ -58,9 +58,9 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
-	golang.org/x/crypto v0.17.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
+	golang.org/x/sys v0.21.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce
@@ -85,6 +85,7 @@ require (
 	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/gotestyourself/gotestyourself v1.4.0 // indirect
 	github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
@@ -96,14 +97,14 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rivo/uniseg v0.4.4 // indirect
-	github.com/supranational/blst v0.3.11-0.20230124161941-ca03e11a3ff2 // indirect
+	github.com/supranational/blst v0.3.11 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/net v0.16.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	google.golang.org/protobuf v1.23.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -38,9 +38,11 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40=
-github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o=
+github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
+github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
+github.com/agiledragon/gomonkey/v2 v2.12.0 h1:ek0dYu9K1rSV+TgkW5LvNNPRWyDZVIxGMCFI6Pz9o38=
+github.com/agiledragon/gomonkey/v2 v2.12.0/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -278,6 +280,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -392,8 +396,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/scroll-tech/da-codec v0.1.1-0.20240822151711-9e32313056ac h1:DjLrqjoOLVFug9ZkAbJYwjtYW51YZE0Num3p4cZXaZs=
-github.com/scroll-tech/da-codec v0.1.1-0.20240822151711-9e32313056ac/go.mod h1:D6XEESeNVJkQJlv3eK+FyR+ufPkgVQbJzERylQi53Bs=
+github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5 h1:vZ75srkZCStjDWq/kqZGLoucf7Y7qXC13nKjQVZ0zp8=
+github.com/scroll-tech/da-codec v0.1.3-0.20241218102542-9852fa4e1be5/go.mod h1:XfQhUl3msmE6dpZEbR/LIwiMxywPQcUQsch9URgXDzs=
 github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE=
 github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk=
 github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
@@ -428,8 +432,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/supranational/blst v0.3.11-0.20230124161941-ca03e11a3ff2 h1:wh1wzwAhZBNiZO37uWS/nDaKiIwHz4mDo4pnA+fqTO0=
-github.com/supranational/blst v0.3.11-0.20230124161941-ca03e11a3ff2/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
+github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
+github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -470,8 +474,8 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -522,8 +526,8 @@ golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos=
-golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -577,14 +581,15 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

miner/miner.go

@@ -76,7 +76,7 @@ type Miner struct {
 	wg sync.WaitGroup
 }
 
-func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(block *types.Block) bool) *Miner {
+func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *event.TypeMux, engine consensus.Engine, isLocalBlock func(block *types.Block) bool, daSyncingEnabled bool) *Miner {
 	miner := &Miner{
 		eth:     eth,
 		mux:     mux,
@@ -84,10 +84,12 @@ func New(eth Backend, config *Config, chainConfig *params.ChainConfig, mux *even
 		exitCh:  make(chan struct{}),
 		startCh: make(chan common.Address),
 		stopCh:  make(chan struct{}),
-		worker:  newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true),
+		worker:  newWorker(config, chainConfig, engine, eth, mux, isLocalBlock, true, daSyncingEnabled),
+	}
+	if !daSyncingEnabled {
+		miner.wg.Add(1)
+		go miner.update()
 	}
-	miner.wg.Add(1)
-	go miner.update()
 	return miner
 }
 

miner/miner_test.go

@@ -276,5 +276,5 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux) {
 	// Create event Mux
 	mux := new(event.TypeMux)
 	// Create Miner
-	return New(backend, &config, chainConfig, mux, engine, nil), mux
+	return New(backend, &config, chainConfig, mux, engine, nil, false), mux
 }

miner/scroll_worker.go

@@ -177,7 +177,7 @@ type worker struct {
 	skipTxHash   common.Hash
 }
 
-func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool) *worker {
+func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus.Engine, eth Backend, mux *event.TypeMux, isLocalBlock func(*types.Block) bool, init bool, daSyncingEnabled bool) *worker {
 	worker := &worker{
 		config:       config,
 		chainConfig:  chainConfig,
@@ -192,6 +192,12 @@ func newWorker(config *Config, chainConfig *params.ChainConfig, engine consensus
 		startCh:      make(chan struct{}, 1),
 		reorgCh:      make(chan reorgTrigger, 1),
 	}
+
+	if daSyncingEnabled {
+		log.Info("Worker will not start, because DA syncing is enabled")
+		return worker
+	}
+
 	worker.asyncChecker = ccc.NewAsyncChecker(worker.chain, config.CCCMaxWorkers, false).WithOnFailingBlock(worker.onBlockFailingCCC)
 
 	// Subscribe NewTxsEvent for tx pool

miner/scroll_worker_test.go

@@ -208,7 +208,7 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction {
 func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) {
 	backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks)
 	backend.txPool.AddLocals(pendingTxs)
-	w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false)
+	w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false, false)
 	w.setEtherbase(testBankAddress)
 	return w, backend
 }

node/config.go

@@ -197,6 +197,8 @@ type Config struct {
 	L1Confirmations rpc.BlockNumber `toml:",omitempty"`
 	// L1 bridge deployment block number
 	L1DeploymentBlock uint64 `toml:",omitempty"`
+	// Is daSyncingEnabled
+	DaSyncingEnabled bool `toml:",omitempty"`
 }
 
 // IPCEndpoint resolves an IPC endpoint based on a configured value, taking into

node/node.go

@@ -262,10 +262,15 @@ func (n *Node) doClose(errs []error) error {
 // openEndpoints starts all network and RPC endpoints.
 func (n *Node) openEndpoints() error {
 	// start networking endpoints
-	n.log.Info("Starting peer-to-peer node", "instance", n.server.Name)
-	if err := n.server.Start(); err != nil {
-		return convertFileLockError(err)
+	if !n.config.DaSyncingEnabled {
+		n.log.Info("Starting peer-to-peer node", "instance", n.server.Name)
+		if err := n.server.Start(); err != nil {
+			return convertFileLockError(err)
+		}
+	} else {
+		n.log.Info("Peer-to-peer node will not start, because DA syncing is enabled")
 	}
+
 	// start RPC endpoints
 	err := n.startRPC()
 	if err != nil {

params/version.go

@@ -23,8 +23,8 @@ import (
 
 const (
 	VersionMajor = 5         // Major version component of the current release
-	VersionMinor = 7         // Minor version component of the current release
-	VersionPatch = 29        // Patch version component of the current release
+	VersionMinor = 8         // Minor version component of the current release
+	VersionPatch = 0         // Patch version component of the current release
 	VersionMeta  = "mainnet" // Version metadata to append to the version string
 )
 

rollup/da_syncer/batch_queue.go

@@ -0,0 +1,103 @@
+package da_syncer
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/core/rawdb"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+)
+
+// BatchQueue is a pipeline stage that reads all batch events from DAQueue and provides only finalized batches to the next stage.
+type BatchQueue struct {
+	DAQueue                 *DAQueue
+	db                      ethdb.Database
+	lastFinalizedBatchIndex uint64
+	batches                 *common.Heap[da.Entry]
+	batchesMap              *common.ShrinkingMap[uint64, *common.HeapElement[da.Entry]]
+}
+
+func NewBatchQueue(DAQueue *DAQueue, db ethdb.Database) *BatchQueue {
+	return &BatchQueue{
+		DAQueue:                 DAQueue,
+		db:                      db,
+		lastFinalizedBatchIndex: 0,
+		batches:                 common.NewHeap[da.Entry](),
+		batchesMap:              common.NewShrinkingMap[uint64, *common.HeapElement[da.Entry]](1000),
+	}
+}
+
+// NextBatch finds next finalized batch and returns data, that was committed in that batch
+func (bq *BatchQueue) NextBatch(ctx context.Context) (da.Entry, error) {
+	if batch := bq.getFinalizedBatch(); batch != nil {
+		return batch, nil
+	}
+
+	for {
+		daEntry, err := bq.DAQueue.NextDA(ctx)
+		if err != nil {
+			return nil, err
+		}
+		switch daEntry.Type() {
+		case da.CommitBatchV0Type, da.CommitBatchWithBlobType:
+			bq.addBatch(daEntry)
+		case da.RevertBatchType:
+			bq.deleteBatch(daEntry)
+		case da.FinalizeBatchType:
+			if daEntry.BatchIndex() > bq.lastFinalizedBatchIndex {
+				bq.lastFinalizedBatchIndex = daEntry.BatchIndex()
+			}
+
+			if batch := bq.getFinalizedBatch(); batch != nil {
+				return batch, nil
+			}
+		default:
+			return nil, fmt.Errorf("unexpected type of daEntry: %T", daEntry)
+		}
+	}
+}
+
+// getFinalizedBatch returns next finalized batch if there is available
+func (bq *BatchQueue) getFinalizedBatch() da.Entry {
+	if bq.batches.Len() == 0 {
+		return nil
+	}
+
+	batch := bq.batches.Peek().Value()
+	if batch.BatchIndex() <= bq.lastFinalizedBatchIndex {
+		bq.deleteBatch(batch)
+		return batch
+	} else {
+		return nil
+	}
+}
+
+func (bq *BatchQueue) addBatch(batch da.Entry) {
+	heapElement := bq.batches.Push(batch)
+	bq.batchesMap.Set(batch.BatchIndex(), heapElement)
+}
+
+// deleteBatch deletes data committed in the batch from map, because this batch is reverted or finalized
+// updates DASyncedL1BlockNumber
+func (bq *BatchQueue) deleteBatch(batch da.Entry) {
+	batchHeapElement, exists := bq.batchesMap.Get(batch.BatchIndex())
+	if !exists {
+		return
+	}
+
+	bq.batchesMap.Delete(batch.BatchIndex())
+	bq.batches.Remove(batchHeapElement)
+
+	// we store here min height of currently loaded batches to be able to start syncing from the same place in case of restart
+	// TODO: we should store this information when the batch is done being processed to avoid inconsistencies
+	rawdb.WriteDASyncedL1BlockNumber(bq.db, batch.L1BlockNumber()-1)
+}
+
+func (bq *BatchQueue) Reset(height uint64) {
+	bq.batches.Clear()
+	bq.batchesMap.Clear()
+	bq.lastFinalizedBatchIndex = 0
+	bq.DAQueue.Reset(height)
+}

rollup/da_syncer/blob_client/beacon_node_client.go

@@ -0,0 +1,192 @@
+package blob_client
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
+)
+
+type BeaconNodeClient struct {
+	apiEndpoint    string
+	l1Client       *rollup_sync_service.L1Client
+	genesisTime    uint64
+	secondsPerSlot uint64
+}
+
+var (
+	beaconNodeGenesisEndpoint = "/eth/v1/beacon/genesis"
+	beaconNodeSpecEndpoint    = "/eth/v1/config/spec"
+	beaconNodeBlobEndpoint    = "/eth/v1/beacon/blob_sidecars"
+)
+
+func NewBeaconNodeClient(apiEndpoint string, l1Client *rollup_sync_service.L1Client) (*BeaconNodeClient, error) {
+	// get genesis time
+	genesisPath, err := url.JoinPath(apiEndpoint, beaconNodeGenesisEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to join path, err: %w", err)
+	}
+	resp, err := http.Get(genesisPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot do request, err: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("beacon node request failed with status: %s: could not read response body: %w", resp.Status, err)
+		}
+		bodyStr := string(body)
+		return nil, fmt.Errorf("beacon node request failed, status: %s, body: %s", resp.Status, bodyStr)
+	}
+
+	var genesisResp GenesisResp
+	err = json.NewDecoder(resp.Body).Decode(&genesisResp)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+	}
+	genesisTime, err := strconv.ParseUint(genesisResp.Data.GenesisTime, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode genesis time %s, err: %w", genesisResp.Data.GenesisTime, err)
+	}
+
+	// get seconds per slot from spec
+	specPath, err := url.JoinPath(apiEndpoint, beaconNodeSpecEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("failed to join path, err: %w", err)
+	}
+	resp, err = http.Get(specPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot do request, err: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("beacon node request failed with status: %s: could not read response body: %w", resp.Status, err)
+		}
+		bodyStr := string(body)
+		return nil, fmt.Errorf("beacon node request failed, status: %s, body: %s", resp.Status, bodyStr)
+	}
+
+	var specResp SpecResp
+	err = json.NewDecoder(resp.Body).Decode(&specResp)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+	}
+	secondsPerSlot, err := strconv.ParseUint(specResp.Data.SecondsPerSlot, 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode seconds per slot %s, err: %w", specResp.Data.SecondsPerSlot, err)
+	}
+	if secondsPerSlot == 0 {
+		return nil, fmt.Errorf("failed to make new BeaconNodeClient, secondsPerSlot is 0")
+	}
+
+	return &BeaconNodeClient{
+		apiEndpoint:    apiEndpoint,
+		l1Client:       l1Client,
+		genesisTime:    genesisTime,
+		secondsPerSlot: secondsPerSlot,
+	}, nil
+}
+
+func (c *BeaconNodeClient) GetBlobByVersionedHashAndBlockNumber(ctx context.Context, versionedHash common.Hash, blockNumber uint64) (*kzg4844.Blob, error) {
+	// get block timestamp to calculate slot
+	header, err := c.l1Client.GetHeaderByNumber(blockNumber)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get header by number, err: %w", err)
+	}
+	slot := (header.Time - c.genesisTime) / c.secondsPerSlot
+
+	// get blob sidecar for slot
+	blobSidecarPath, err := url.JoinPath(c.apiEndpoint, beaconNodeBlobEndpoint, fmt.Sprintf("%d", slot))
+	if err != nil {
+		return nil, fmt.Errorf("failed to join path, err: %w", err)
+	}
+	resp, err := http.Get(blobSidecarPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot do request, err: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return nil, fmt.Errorf("beacon node request failed with status: %s: could not read response body: %w", resp.Status, err)
+		}
+		bodyStr := string(body)
+		return nil, fmt.Errorf("beacon node request failed, status: %s, body: %s", resp.Status, bodyStr)
+	}
+
+	var blobSidecarResp BlobSidecarResp
+	err = json.NewDecoder(resp.Body).Decode(&blobSidecarResp)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+	}
+
+	// find blob with desired versionedHash
+	for _, blob := range blobSidecarResp.Data {
+		// calculate blob hash from commitment and check it with desired
+		commitmentBytes := common.FromHex(blob.KzgCommitment)
+		if len(commitmentBytes) != lenKZGCommitment {
+			return nil, fmt.Errorf("len of kzg commitment is not correct, expected: %d, got: %d", lenKZGCommitment, len(commitmentBytes))
+		}
+		commitment := kzg4844.Commitment(commitmentBytes)
+		blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &commitment)
+
+		if blobVersionedHash == versionedHash {
+			// found desired blob
+			blobBytes := common.FromHex(blob.Blob)
+			if len(blobBytes) != lenBlobBytes {
+				return nil, fmt.Errorf("len of blob data is not correct, expected: %d, got: %d", lenBlobBytes, len(blobBytes))
+			}
+
+			b := kzg4844.Blob(blobBytes)
+			return &b, nil
+		}
+	}
+
+	return nil, fmt.Errorf("missing blob %v in slot %d, block number %d", versionedHash, slot, blockNumber)
+}
+
+type GenesisResp struct {
+	Data struct {
+		GenesisTime string `json:"genesis_time"`
+	} `json:"data"`
+}
+
+type SpecResp struct {
+	Data struct {
+		SecondsPerSlot string `json:"SECONDS_PER_SLOT"`
+	} `json:"data"`
+}
+
+type BlobSidecarResp struct {
+	Data []struct {
+		Index             string `json:"index"`
+		Blob              string `json:"blob"`
+		KzgCommitment     string `json:"kzg_commitment"`
+		KzgProof          string `json:"kzg_proof"`
+		SignedBlockHeader struct {
+			Message struct {
+				Slot          string `json:"slot"`
+				ProposerIndex string `json:"proposer_index"`
+				ParentRoot    string `json:"parent_root"`
+				StateRoot     string `json:"state_root"`
+				BodyRoot      string `json:"body_root"`
+			} `json:"message"`
+			Signature string `json:"signature"`
+		} `json:"signed_block_header"`
+		KzgCommitmentInclusionProof []string `json:"kzg_commitment_inclusion_proof"`
+	} `json:"data"`
+}

rollup/da_syncer/blob_client/blob_client.go

@@ -0,0 +1,64 @@
+package blob_client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
+)
+
+const (
+	lenBlobBytes     int = 131072
+	lenKZGCommitment int = 48
+)
+
+type BlobClient interface {
+	GetBlobByVersionedHashAndBlockNumber(ctx context.Context, versionedHash common.Hash, blockNumber uint64) (*kzg4844.Blob, error)
+}
+
+type BlobClients struct {
+	list   []BlobClient
+	curPos int
+}
+
+func NewBlobClients(blobClients ...BlobClient) *BlobClients {
+	return &BlobClients{
+		list:   blobClients,
+		curPos: 0,
+	}
+}
+
+func (c *BlobClients) GetBlobByVersionedHashAndBlockNumber(ctx context.Context, versionedHash common.Hash, blockNumber uint64) (*kzg4844.Blob, error) {
+	if len(c.list) == 0 {
+		return nil, fmt.Errorf("BlobClients.GetBlobByVersionedHash: list of BlobClients is empty")
+	}
+
+	for i := 0; i < len(c.list); i++ {
+		blob, err := c.list[c.curPos].GetBlobByVersionedHashAndBlockNumber(ctx, versionedHash, blockNumber)
+		if err == nil {
+			return blob, nil
+		}
+		c.nextPos()
+		// there was an error, try the next blob client in following iteration
+		log.Warn("BlobClients: failed to get blob by versioned hash from BlobClient", "err", err, "blob client pos in BlobClients", c.curPos)
+	}
+
+	// if we iterated over entire list, return a temporary error that will be handled in syncing_pipeline with a backoff and retry
+	return nil, serrors.NewTemporaryError(errors.New("BlobClients.GetBlobByVersionedHash: failed to get blob by versioned hash from all BlobClients"))
+}
+
+func (c *BlobClients) nextPos() {
+	c.curPos = (c.curPos + 1) % len(c.list)
+}
+
+func (c *BlobClients) AddBlobClient(blobClient BlobClient) {
+	c.list = append(c.list, blobClient)
+}
+
+func (c *BlobClients) Size() int {
+	return len(c.list)
+}

rollup/da_syncer/blob_client/blob_scan_client.go

@@ -0,0 +1,92 @@
+package blob_client
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/common/hexutil"
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+type BlobScanClient struct {
+	client      *http.Client
+	apiEndpoint string
+}
+
+func NewBlobScanClient(apiEndpoint string) *BlobScanClient {
+	return &BlobScanClient{
+		client:      http.DefaultClient,
+		apiEndpoint: apiEndpoint,
+	}
+}
+
+func (c *BlobScanClient) GetBlobByVersionedHashAndBlockNumber(ctx context.Context, versionedHash common.Hash, blockNumber uint64) (*kzg4844.Blob, error) {
+	// blobscan api docs https://api.blobscan.com/#/blobs/blob-getByBlobId
+	path, err := url.JoinPath(c.apiEndpoint, versionedHash.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to join path, err: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request, err: %w", err)
+	}
+	req.Header.Set("accept", "application/json")
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot do request, err: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			return nil, fmt.Errorf("no blob with versioned hash : %s", versionedHash.String())
+		}
+		var res ErrorRespBlobScan
+		err = json.NewDecoder(resp.Body).Decode(&res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+		}
+		return nil, fmt.Errorf("error while fetching blob, message: %s, code: %s, versioned hash: %s", res.Message, res.Code, versionedHash.String())
+	}
+	var result BlobRespBlobScan
+
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+	}
+	blobBytes, err := hex.DecodeString(result.Data[2:])
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode data to bytes, err: %w", err)
+	}
+	if len(blobBytes) != lenBlobBytes {
+		return nil, fmt.Errorf("len of blob data is not correct, expected: %d, got: %d", lenBlobBytes, len(blobBytes))
+	}
+	blob := kzg4844.Blob(blobBytes)
+
+	// sanity check that retrieved blob matches versioned hash
+	commitment, err := kzg4844.BlobToCommitment(&blob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert blob to commitment, err: %w", err)
+	}
+
+	blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &commitment)
+	if blobVersionedHash != versionedHash {
+		return nil, fmt.Errorf("blob versioned hash mismatch, expected: %s, got: %s", versionedHash.String(), hexutil.Encode(blobVersionedHash[:]))
+	}
+
+	return &blob, nil
+}
+
+type BlobRespBlobScan struct {
+	Data string `json:"data"`
+}
+
+type ErrorRespBlobScan struct {
+	Message string `json:"message"`
+	Code    string `json:"code"`
+}

rollup/da_syncer/blob_client/block_native_client.go

@@ -0,0 +1,89 @@
+package blob_client
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/common/hexutil"
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+)
+
+type BlockNativeClient struct {
+	apiEndpoint string
+}
+
+func NewBlockNativeClient(apiEndpoint string) *BlockNativeClient {
+	return &BlockNativeClient{
+		apiEndpoint: apiEndpoint,
+	}
+}
+
+func (c *BlockNativeClient) GetBlobByVersionedHashAndBlockNumber(ctx context.Context, versionedHash common.Hash, blockNumber uint64) (*kzg4844.Blob, error) {
+	// blocknative api docs https://docs.blocknative.com/blocknative-data-archive/blob-archive
+	path, err := url.JoinPath(c.apiEndpoint, versionedHash.String())
+	if err != nil {
+		return nil, fmt.Errorf("failed to join path, err: %w", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, "GET", path, nil)
+	if err != nil {
+		return nil, fmt.Errorf("cannot create request, err: %w", err)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("cannot do request, err: %w", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		var res ErrorRespBlockNative
+		err = json.NewDecoder(resp.Body).Decode(&res)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+		}
+		return nil, fmt.Errorf("error while fetching blob, message: %s, code: %d, versioned hash: %s", res.Error.Message, res.Error.Code, versionedHash.String())
+	}
+	var result BlobRespBlockNative
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode result into struct, err: %w", err)
+	}
+	blobBytes, err := hex.DecodeString(result.Blob.Data[2:])
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode data to bytes, err: %w", err)
+	}
+	if len(blobBytes) != lenBlobBytes {
+		return nil, fmt.Errorf("len of blob data is not correct, expected: %d, got: %d", lenBlobBytes, len(blobBytes))
+	}
+	blob := kzg4844.Blob(blobBytes)
+
+	// sanity check that retrieved blob matches versioned hash
+	commitment, err := kzg4844.BlobToCommitment(&blob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert blob to commitment, err: %w", err)
+	}
+
+	blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &commitment)
+	if blobVersionedHash != versionedHash {
+		return nil, fmt.Errorf("blob versioned hash mismatch, expected: %s, got: %s", versionedHash.String(), hexutil.Encode(blobVersionedHash[:]))
+	}
+
+	return &blob, nil
+}
+
+type BlobRespBlockNative struct {
+	Blob struct {
+		Data string `json:"data"`
+	} `json:"blob"`
+}
+
+type ErrorRespBlockNative struct {
+	Error struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error"`
+}

rollup/da_syncer/block_queue.go

@@ -0,0 +1,56 @@
+package da_syncer
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+)
+
+// BlockQueue is a pipeline stage that reads batches from BatchQueue, extracts all da.PartialBlock from it and
+// provides them to the next stage one-by-one.
+type BlockQueue struct {
+	batchQueue *BatchQueue
+	blocks     []*da.PartialBlock
+}
+
+func NewBlockQueue(batchQueue *BatchQueue) *BlockQueue {
+	return &BlockQueue{
+		batchQueue: batchQueue,
+		blocks:     make([]*da.PartialBlock, 0),
+	}
+}
+
+func (bq *BlockQueue) NextBlock(ctx context.Context) (*da.PartialBlock, error) {
+	for len(bq.blocks) == 0 {
+		err := bq.getBlocksFromBatch(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	block := bq.blocks[0]
+	bq.blocks = bq.blocks[1:]
+	return block, nil
+}
+
+func (bq *BlockQueue) getBlocksFromBatch(ctx context.Context) error {
+	daEntry, err := bq.batchQueue.NextBatch(ctx)
+	if err != nil {
+		return err
+	}
+
+	entryWithBlocks, ok := daEntry.(da.EntryWithBlocks)
+	// this should never happen because we only receive CommitBatch entries
+	if !ok {
+		return fmt.Errorf("unexpected type of daEntry: %T", daEntry)
+	}
+
+	bq.blocks = entryWithBlocks.Blocks()
+
+	return nil
+}
+
+func (bq *BlockQueue) Reset(height uint64) {
+	bq.blocks = make([]*da.PartialBlock, 0)
+	bq.batchQueue.Reset(height)
+}

rollup/da_syncer/da/calldata_blob_source.go

@@ -0,0 +1,247 @@
+package da
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/scroll-tech/da-codec/encoding"
+
+	"github.com/scroll-tech/go-ethereum/accounts/abi"
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
+	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
+)
+
+const (
+	callDataBlobSourceFetchBlockRange  uint64 = 500
+	commitBatchEventName                      = "CommitBatch"
+	revertBatchEventName                      = "RevertBatch"
+	finalizeBatchEventName                    = "FinalizeBatch"
+	commitBatchMethodName                     = "commitBatch"
+	commitBatchWithBlobProofMethodName        = "commitBatchWithBlobProof"
+
+	// the length of method ID at the beginning of transaction data
+	methodIDLength = 4
+)
+
+var (
+	ErrSourceExhausted = errors.New("data source has been exhausted")
+)
+
+type CalldataBlobSource struct {
+	ctx                           context.Context
+	l1Client                      *rollup_sync_service.L1Client
+	blobClient                    blob_client.BlobClient
+	l1height                      uint64
+	scrollChainABI                *abi.ABI
+	l1CommitBatchEventSignature   common.Hash
+	l1RevertBatchEventSignature   common.Hash
+	l1FinalizeBatchEventSignature common.Hash
+	db                            ethdb.Database
+
+	l1Finalized uint64
+}
+
+func NewCalldataBlobSource(ctx context.Context, l1height uint64, l1Client *rollup_sync_service.L1Client, blobClient blob_client.BlobClient, db ethdb.Database) (*CalldataBlobSource, error) {
+	scrollChainABI, err := rollup_sync_service.ScrollChainMetaData.GetAbi()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get scroll chain abi: %w", err)
+	}
+	return &CalldataBlobSource{
+		ctx:                           ctx,
+		l1Client:                      l1Client,
+		blobClient:                    blobClient,
+		l1height:                      l1height,
+		scrollChainABI:                scrollChainABI,
+		l1CommitBatchEventSignature:   scrollChainABI.Events[commitBatchEventName].ID,
+		l1RevertBatchEventSignature:   scrollChainABI.Events[revertBatchEventName].ID,
+		l1FinalizeBatchEventSignature: scrollChainABI.Events[finalizeBatchEventName].ID,
+		db:                            db,
+	}, nil
+}
+
+func (ds *CalldataBlobSource) NextData() (Entries, error) {
+	var err error
+	to := ds.l1height + callDataBlobSourceFetchBlockRange
+
+	// If there's not enough finalized blocks to request up to, we need to query finalized block number.
+	// Otherwise, we know that there's more finalized blocks than we want to request up to
+	// -> no need to query finalized block number
+	if to > ds.l1Finalized {
+		ds.l1Finalized, err = ds.l1Client.GetLatestFinalizedBlockNumber()
+		if err != nil {
+			return nil, serrors.NewTemporaryError(fmt.Errorf("failed to query GetLatestFinalizedBlockNumber, error: %v", err))
+		}
+		// make sure we don't request more than finalized blocks
+		to = min(to, ds.l1Finalized)
+	}
+
+	if ds.l1height > to {
+		return nil, ErrSourceExhausted
+	}
+
+	logs, err := ds.l1Client.FetchRollupEventsInRange(ds.l1height, to)
+	if err != nil {
+		return nil, serrors.NewTemporaryError(fmt.Errorf("cannot get events, l1height: %d, error: %v", ds.l1height, err))
+	}
+	da, err := ds.processLogsToDA(logs)
+	if err != nil {
+		return nil, serrors.NewTemporaryError(fmt.Errorf("failed to process logs to DA, error: %v", err))
+	}
+
+	ds.l1height = to + 1
+	return da, nil
+}
+
+func (ds *CalldataBlobSource) L1Height() uint64 {
+	return ds.l1height
+}
+
+func (ds *CalldataBlobSource) processLogsToDA(logs []types.Log) (Entries, error) {
+	var entries Entries
+	var entry Entry
+	var err error
+
+	for _, vLog := range logs {
+		switch vLog.Topics[0] {
+		case ds.l1CommitBatchEventSignature:
+			event := &rollup_sync_service.L1CommitBatchEvent{}
+			if err = rollup_sync_service.UnpackLog(ds.scrollChainABI, event, commitBatchEventName, vLog); err != nil {
+				return nil, fmt.Errorf("failed to unpack commit rollup event log, err: %w", err)
+			}
+
+			batchIndex := event.BatchIndex.Uint64()
+			log.Trace("found new CommitBatch event", "batch index", batchIndex)
+
+			if entry, err = ds.getCommitBatchDA(batchIndex, &vLog); err != nil {
+				return nil, fmt.Errorf("failed to get commit batch da: %v, err: %w", batchIndex, err)
+			}
+
+		case ds.l1RevertBatchEventSignature:
+			event := &rollup_sync_service.L1RevertBatchEvent{}
+			if err = rollup_sync_service.UnpackLog(ds.scrollChainABI, event, revertBatchEventName, vLog); err != nil {
+				return nil, fmt.Errorf("failed to unpack revert rollup event log, err: %w", err)
+			}
+
+			batchIndex := event.BatchIndex.Uint64()
+			log.Trace("found new RevertBatchType event", "batch index", batchIndex)
+			entry = NewRevertBatch(batchIndex)
+
+		case ds.l1FinalizeBatchEventSignature:
+			event := &rollup_sync_service.L1FinalizeBatchEvent{}
+			if err = rollup_sync_service.UnpackLog(ds.scrollChainABI, event, finalizeBatchEventName, vLog); err != nil {
+				return nil, fmt.Errorf("failed to unpack finalized rollup event log, err: %w", err)
+			}
+
+			batchIndex := event.BatchIndex.Uint64()
+			log.Trace("found new FinalizeBatchType event", "batch index", event.BatchIndex.Uint64())
+			entry = NewFinalizeBatch(batchIndex)
+
+		default:
+			return nil, fmt.Errorf("unknown event, topic: %v, tx hash: %v", vLog.Topics[0].Hex(), vLog.TxHash.Hex())
+		}
+
+		entries = append(entries, entry)
+	}
+	return entries, nil
+}
+
+type commitBatchArgs struct {
+	Version                uint8
+	ParentBatchHeader      []byte
+	Chunks                 [][]byte
+	SkippedL1MessageBitmap []byte
+}
+
+func newCommitBatchArgs(method *abi.Method, values []interface{}) (*commitBatchArgs, error) {
+	var args commitBatchArgs
+	err := method.Inputs.Copy(&args, values)
+	return &args, err
+}
+
+func newCommitBatchArgsFromCommitBatchWithProof(method *abi.Method, values []interface{}) (*commitBatchArgs, error) {
+	var args commitBatchWithBlobProofArgs
+	err := method.Inputs.Copy(&args, values)
+	if err != nil {
+		return nil, err
+	}
+	return &commitBatchArgs{
+		Version:                args.Version,
+		ParentBatchHeader:      args.ParentBatchHeader,
+		Chunks:                 args.Chunks,
+		SkippedL1MessageBitmap: args.SkippedL1MessageBitmap,
+	}, nil
+}
+
+type commitBatchWithBlobProofArgs struct {
+	Version                uint8
+	ParentBatchHeader      []byte
+	Chunks                 [][]byte
+	SkippedL1MessageBitmap []byte
+	BlobDataProof          []byte
+}
+
+func (ds *CalldataBlobSource) getCommitBatchDA(batchIndex uint64, vLog *types.Log) (Entry, error) {
+	if batchIndex == 0 {
+		return NewCommitBatchDAV0Empty(), nil
+	}
+
+	txData, err := ds.l1Client.FetchTxData(vLog)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch tx data, tx hash: %v, err: %w", vLog.TxHash.Hex(), err)
+	}
+	if len(txData) < methodIDLength {
+		return nil, fmt.Errorf("transaction data is too short, length of tx data: %v, minimum length required: %v", len(txData), methodIDLength)
+	}
+
+	method, err := ds.scrollChainABI.MethodById(txData[:methodIDLength])
+	if err != nil {
+		return nil, fmt.Errorf("failed to get method by ID, ID: %v, err: %w", txData[:methodIDLength], err)
+	}
+	values, err := method.Inputs.Unpack(txData[methodIDLength:])
+	if err != nil {
+		return nil, fmt.Errorf("failed to unpack transaction data using ABI, tx data: %v, err: %w", txData, err)
+	}
+	if method.Name == commitBatchMethodName {
+		args, err := newCommitBatchArgs(method, values)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode calldata into commitBatch args, values: %+v, err: %w", values, err)
+		}
+		codecVersion := encoding.CodecVersion(args.Version)
+		codec, err := encoding.CodecFromVersion(codecVersion)
+		if err != nil {
+			return nil, fmt.Errorf("unsupported codec version: %v, batch index: %v, err: %w", codecVersion, batchIndex, err)
+		}
+		switch args.Version {
+		case 0:
+			return NewCommitBatchDAV0(ds.db, codec, args.Version, batchIndex, args.ParentBatchHeader, args.Chunks, args.SkippedL1MessageBitmap, vLog.BlockNumber)
+		case 1, 2:
+			return NewCommitBatchDAWithBlob(ds.ctx, ds.db, codec, ds.l1Client, ds.blobClient, vLog, args.Version, batchIndex, args.ParentBatchHeader, args.Chunks, args.SkippedL1MessageBitmap)
+		default:
+			return nil, fmt.Errorf("failed to decode DA, codec version is unknown: codec version: %d", args.Version)
+		}
+	} else if method.Name == commitBatchWithBlobProofMethodName {
+		args, err := newCommitBatchArgsFromCommitBatchWithProof(method, values)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode calldata into commitBatch args, values: %+v, err: %w", values, err)
+		}
+		codecVersion := encoding.CodecVersion(args.Version)
+		codec, err := encoding.CodecFromVersion(codecVersion)
+		if err != nil {
+			return nil, fmt.Errorf("unsupported codec version: %v, batch index: %v, err: %w", codecVersion, batchIndex, err)
+		}
+		switch args.Version {
+		case 3, 4:
+			return NewCommitBatchDAWithBlob(ds.ctx, ds.db, codec, ds.l1Client, ds.blobClient, vLog, args.Version, batchIndex, args.ParentBatchHeader, args.Chunks, args.SkippedL1MessageBitmap)
+		default:
+			return nil, fmt.Errorf("failed to decode DA, codec version is unknown: codec version: %d", args.Version)
+		}
+	}
+
+	return nil, fmt.Errorf("unknown method name: %s", method.Name)
+}

rollup/da_syncer/da/commitV0.go

@@ -0,0 +1,172 @@
+package da
+
+import (
+	"encoding/binary"
+	"fmt"
+
+	"github.com/scroll-tech/da-codec/encoding"
+
+	"github.com/scroll-tech/go-ethereum/core/rawdb"
+	"github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
+)
+
+type CommitBatchDAV0 struct {
+	version                    uint8
+	batchIndex                 uint64
+	parentTotalL1MessagePopped uint64
+	skippedL1MessageBitmap     []byte
+	chunks                     []*encoding.DAChunkRawTx
+	l1Txs                      []*types.L1MessageTx
+
+	l1BlockNumber uint64
+}
+
+func NewCommitBatchDAV0(db ethdb.Database,
+	codec encoding.Codec,
+	version uint8,
+	batchIndex uint64,
+	parentBatchHeader []byte,
+	chunks [][]byte,
+	skippedL1MessageBitmap []byte,
+	l1BlockNumber uint64,
+) (*CommitBatchDAV0, error) {
+	decodedChunks, err := codec.DecodeDAChunksRawTx(chunks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unpack chunks: %d, err: %w", batchIndex, err)
+	}
+
+	return NewCommitBatchDAV0WithChunks(db, version, batchIndex, parentBatchHeader, decodedChunks, skippedL1MessageBitmap, l1BlockNumber)
+}
+
+func NewCommitBatchDAV0WithChunks(db ethdb.Database,
+	version uint8,
+	batchIndex uint64,
+	parentBatchHeader []byte,
+	decodedChunks []*encoding.DAChunkRawTx,
+	skippedL1MessageBitmap []byte,
+	l1BlockNumber uint64,
+) (*CommitBatchDAV0, error) {
+	parentTotalL1MessagePopped := getBatchTotalL1MessagePopped(parentBatchHeader)
+	l1Txs, err := getL1Messages(db, parentTotalL1MessagePopped, skippedL1MessageBitmap, getTotalMessagesPoppedFromChunks(decodedChunks))
+	if err != nil {
+		return nil, fmt.Errorf("failed to get L1 messages for v0 batch %d: %w", batchIndex, err)
+	}
+
+	return &CommitBatchDAV0{
+		version:                    version,
+		batchIndex:                 batchIndex,
+		parentTotalL1MessagePopped: parentTotalL1MessagePopped,
+		skippedL1MessageBitmap:     skippedL1MessageBitmap,
+		chunks:                     decodedChunks,
+		l1Txs:                      l1Txs,
+		l1BlockNumber:              l1BlockNumber,
+	}, nil
+}
+
+func NewCommitBatchDAV0Empty() *CommitBatchDAV0 {
+	return &CommitBatchDAV0{
+		batchIndex: 0,
+	}
+}
+
+func (c *CommitBatchDAV0) Type() Type {
+	return CommitBatchV0Type
+}
+
+func (c *CommitBatchDAV0) L1BlockNumber() uint64 {
+	return c.l1BlockNumber
+}
+
+func (c *CommitBatchDAV0) BatchIndex() uint64 {
+	return c.batchIndex
+}
+
+func (c *CommitBatchDAV0) CompareTo(other Entry) int {
+	if c.BatchIndex() < other.BatchIndex() {
+		return -1
+	} else if c.BatchIndex() > other.BatchIndex() {
+		return 1
+	}
+	return 0
+}
+
+func (c *CommitBatchDAV0) Blocks() []*PartialBlock {
+	var blocks []*PartialBlock
+	l1TxPointer := 0
+
+	curL1TxIndex := c.parentTotalL1MessagePopped
+	for _, chunk := range c.chunks {
+		for blockId, daBlock := range chunk.Blocks {
+			// create txs
+			txs := make(types.Transactions, 0, daBlock.NumTransactions())
+			// insert l1 msgs
+			for l1TxPointer < len(c.l1Txs) && c.l1Txs[l1TxPointer].QueueIndex < curL1TxIndex+uint64(daBlock.NumL1Messages()) {
+				l1Tx := types.NewTx(c.l1Txs[l1TxPointer])
+				txs = append(txs, l1Tx)
+				l1TxPointer++
+			}
+			curL1TxIndex += uint64(daBlock.NumL1Messages())
+
+			// insert l2 txs
+			txs = append(txs, chunk.Transactions[blockId]...)
+
+			block := NewPartialBlock(
+				&PartialHeader{
+					Number:     daBlock.Number(),
+					Time:       daBlock.Timestamp(),
+					BaseFee:    daBlock.BaseFee(),
+					GasLimit:   daBlock.GasLimit(),
+					Difficulty: 10,                             // TODO: replace with real difficulty
+					ExtraData:  []byte{1, 2, 3, 4, 5, 6, 7, 8}, // TODO: replace with real extra data
+				},
+				txs)
+			blocks = append(blocks, block)
+		}
+	}
+
+	return blocks
+}
+
+func getTotalMessagesPoppedFromChunks(decodedChunks []*encoding.DAChunkRawTx) int {
+	totalL1MessagePopped := 0
+	for _, chunk := range decodedChunks {
+		for _, block := range chunk.Blocks {
+			totalL1MessagePopped += int(block.NumL1Messages())
+		}
+	}
+	return totalL1MessagePopped
+}
+
+func getL1Messages(db ethdb.Database, parentTotalL1MessagePopped uint64, skippedBitmap []byte, totalL1MessagePopped int) ([]*types.L1MessageTx, error) {
+	var txs []*types.L1MessageTx
+	decodedSkippedBitmap, err := encoding.DecodeBitmap(skippedBitmap, totalL1MessagePopped)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode skipped message bitmap: err: %w", err)
+	}
+
+	// get all necessary l1 messages without skipped
+	currentIndex := parentTotalL1MessagePopped
+	for index := 0; index < totalL1MessagePopped; index++ {
+		if encoding.IsL1MessageSkipped(decodedSkippedBitmap, currentIndex-parentTotalL1MessagePopped) {
+			currentIndex++
+			continue
+		}
+		l1Tx := rawdb.ReadL1Message(db, currentIndex)
+		if l1Tx == nil {
+			// message not yet available
+			// we return serrors.EOFError as this will be handled in the syncing pipeline with a backoff and retry
+			return nil, serrors.EOFError
+		}
+		txs = append(txs, l1Tx)
+		currentIndex++
+	}
+
+	return txs, nil
+}
+
+func getBatchTotalL1MessagePopped(data []byte) uint64 {
+	// total l1 message popped stored in bytes from 17 to 24, accordingly to codec spec
+	return binary.BigEndian.Uint64(data[17:25])
+}

rollup/da_syncer/da/commitV1.go

@@ -0,0 +1,82 @@
+package da
+
+import (
+	"context"
+	"crypto/sha256"
+	"fmt"
+
+	"github.com/scroll-tech/da-codec/encoding"
+
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
+	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
+
+	"github.com/scroll-tech/go-ethereum/common"
+	"github.com/scroll-tech/go-ethereum/core/types"
+	"github.com/scroll-tech/go-ethereum/crypto/kzg4844"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+)
+
+type CommitBatchDAV1 struct {
+	*CommitBatchDAV0
+}
+
+func NewCommitBatchDAWithBlob(ctx context.Context, db ethdb.Database,
+	codec encoding.Codec,
+	l1Client *rollup_sync_service.L1Client,
+	blobClient blob_client.BlobClient,
+	vLog *types.Log,
+	version uint8,
+	batchIndex uint64,
+	parentBatchHeader []byte,
+	chunks [][]byte,
+	skippedL1MessageBitmap []byte,
+) (*CommitBatchDAV1, error) {
+	decodedChunks, err := codec.DecodeDAChunksRawTx(chunks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unpack chunks: %v, err: %w", batchIndex, err)
+	}
+
+	versionedHash, err := l1Client.FetchTxBlobHash(vLog)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch blob hash, err: %w", err)
+	}
+
+	blob, err := blobClient.GetBlobByVersionedHashAndBlockNumber(ctx, versionedHash, vLog.BlockNumber)
+	if err != nil {
+		return nil, fmt.Errorf("failed to fetch blob from blob client, err: %w", err)
+	}
+	if blob == nil {
+		return nil, fmt.Errorf("unexpected, blob == nil and err != nil, batch index: %d, versionedHash: %s, blobClient: %T", batchIndex, versionedHash.String(), blobClient)
+	}
+
+	// compute blob versioned hash and compare with one from tx
+	c, err := kzg4844.BlobToCommitment(blob)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create blob commitment: %w", err)
+	}
+	blobVersionedHash := common.Hash(kzg4844.CalcBlobHashV1(sha256.New(), &c))
+	if blobVersionedHash != versionedHash {
+		return nil, fmt.Errorf("blobVersionedHash from blob source is not equal to versionedHash from tx, correct versioned hash: %s, fetched blob hash: %s", versionedHash.String(), blobVersionedHash.String())
+	}
+
+	// decode txs from blob
+	err = codec.DecodeTxsFromBlob(blob, decodedChunks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode txs from blob: %w", err)
+	}
+
+	if decodedChunks == nil {
+		return nil, fmt.Errorf("decodedChunks is nil after decoding")
+	}
+
+	v0, err := NewCommitBatchDAV0WithChunks(db, version, batchIndex, parentBatchHeader, decodedChunks, skippedL1MessageBitmap, vLog.BlockNumber)
+	if err != nil {
+		return nil, err
+	}
+
+	return &CommitBatchDAV1{v0}, nil
+}
+
+func (c *CommitBatchDAV1) Type() Type {
+	return CommitBatchWithBlobType
+}

rollup/da_syncer/da/da.go

@@ -0,0 +1,69 @@
+package da
+
+import (
+	"math/big"
+
+	"github.com/scroll-tech/go-ethereum/core/types"
+)
+
+type Type int
+
+const (
+	// CommitBatchV0Type contains data of event of CommitBatchV0Type
+	CommitBatchV0Type Type = iota
+	// CommitBatchWithBlobType contains data of event of CommitBatchWithBlobType (v1, v2, v3, v4)
+	CommitBatchWithBlobType
+	// RevertBatchType contains data of event of RevertBatchType
+	RevertBatchType
+	// FinalizeBatchType contains data of event of FinalizeBatchType
+	FinalizeBatchType
+)
+
+// Entry represents a single DA event (commit, revert, finalize).
+type Entry interface {
+	Type() Type
+	BatchIndex() uint64
+	L1BlockNumber() uint64
+	CompareTo(Entry) int
+}
+
+type EntryWithBlocks interface {
+	Entry
+	Blocks() []*PartialBlock
+}
+
+type Entries []Entry
+
+// PartialHeader represents a partial header (from DA) of a block.
+type PartialHeader struct {
+	Number     uint64
+	Time       uint64
+	BaseFee    *big.Int
+	GasLimit   uint64
+	Difficulty uint64
+	ExtraData  []byte
+}
+
+func (h *PartialHeader) ToHeader() *types.Header {
+	return &types.Header{
+		Number:     big.NewInt(0).SetUint64(h.Number),
+		Time:       h.Time,
+		BaseFee:    h.BaseFee,
+		GasLimit:   h.GasLimit,
+		Difficulty: new(big.Int).SetUint64(h.Difficulty),
+		Extra:      h.ExtraData,
+	}
+}
+
+// PartialBlock represents a partial block (from DA).
+type PartialBlock struct {
+	PartialHeader *PartialHeader
+	Transactions  types.Transactions
+}
+
+func NewPartialBlock(partialHeader *PartialHeader, txs types.Transactions) *PartialBlock {
+	return &PartialBlock{
+		PartialHeader: partialHeader,
+		Transactions:  txs,
+	}
+}

rollup/da_syncer/da/finalize.go

@@ -0,0 +1,34 @@
+package da
+
+type FinalizeBatch struct {
+	batchIndex uint64
+
+	l1BlockNumber uint64
+}
+
+func NewFinalizeBatch(batchIndex uint64) *FinalizeBatch {
+	return &FinalizeBatch{
+		batchIndex: batchIndex,
+	}
+}
+
+func (f *FinalizeBatch) Type() Type {
+	return FinalizeBatchType
+}
+
+func (f *FinalizeBatch) L1BlockNumber() uint64 {
+	return f.l1BlockNumber
+}
+
+func (f *FinalizeBatch) BatchIndex() uint64 {
+	return f.batchIndex
+}
+
+func (f *FinalizeBatch) CompareTo(other Entry) int {
+	if f.BatchIndex() < other.BatchIndex() {
+		return -1
+	} else if f.BatchIndex() > other.BatchIndex() {
+		return 1
+	}
+	return 0
+}

rollup/da_syncer/da/revert.go

@@ -0,0 +1,33 @@
+package da
+
+type RevertBatch struct {
+	batchIndex uint64
+
+	l1BlockNumber uint64
+}
+
+func NewRevertBatch(batchIndex uint64) *RevertBatch {
+	return &RevertBatch{
+		batchIndex: batchIndex,
+	}
+}
+
+func (r *RevertBatch) Type() Type {
+	return RevertBatchType
+}
+
+func (r *RevertBatch) L1BlockNumber() uint64 {
+	return r.l1BlockNumber
+}
+func (r *RevertBatch) BatchIndex() uint64 {
+	return r.batchIndex
+}
+
+func (r *RevertBatch) CompareTo(other Entry) int {
+	if r.BatchIndex() < other.BatchIndex() {
+		return -1
+	} else if r.BatchIndex() > other.BatchIndex() {
+		return 1
+	}
+	return 0
+}

rollup/da_syncer/da_queue.go

@@ -0,0 +1,76 @@
+package da_syncer
+
+import (
+	"context"
+	"errors"
+
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
+)
+
+// DAQueue is a pipeline stage that reads DA entries from a DataSource and provides them to the next stage.
+type DAQueue struct {
+	l1height          uint64
+	dataSourceFactory *DataSourceFactory
+	dataSource        DataSource
+	da                da.Entries
+}
+
+func NewDAQueue(l1height uint64, dataSourceFactory *DataSourceFactory) *DAQueue {
+	return &DAQueue{
+		l1height:          l1height,
+		dataSourceFactory: dataSourceFactory,
+		dataSource:        nil,
+		da:                make(da.Entries, 0),
+	}
+}
+
+func (dq *DAQueue) NextDA(ctx context.Context) (da.Entry, error) {
+	for len(dq.da) == 0 {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		default:
+		}
+
+		err := dq.getNextData(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+	daEntry := dq.da[0]
+	dq.da = dq.da[1:]
+	return daEntry, nil
+}
+
+func (dq *DAQueue) getNextData(ctx context.Context) error {
+	var err error
+	if dq.dataSource == nil {
+		dq.dataSource, err = dq.dataSourceFactory.OpenDataSource(ctx, dq.l1height)
+		if err != nil {
+			return err
+		}
+	}
+
+	dq.da, err = dq.dataSource.NextData()
+	if err == nil {
+		return nil
+	}
+
+	// previous dataSource has been exhausted, create new
+	if errors.Is(err, da.ErrSourceExhausted) {
+		dq.l1height = dq.dataSource.L1Height()
+		dq.dataSource = nil
+
+		// we return EOFError to be handled in pipeline
+		return serrors.EOFError
+	}
+
+	return err
+}
+
+func (dq *DAQueue) Reset(height uint64) {
+	dq.l1height = height
+	dq.dataSource = nil
+	dq.da = make(da.Entries, 0)
+}

rollup/da_syncer/da_syncer.go

@@ -0,0 +1,53 @@
+package da_syncer
+
+import (
+	"fmt"
+
+	"github.com/scroll-tech/go-ethereum/core"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+)
+
+var (
+	ErrBlockTooLow  = fmt.Errorf("block number is too low")
+	ErrBlockTooHigh = fmt.Errorf("block number is too high")
+)
+
+type DASyncer struct {
+	blockchain *core.BlockChain
+}
+
+func NewDASyncer(blockchain *core.BlockChain) *DASyncer {
+	return &DASyncer{
+		blockchain: blockchain,
+	}
+}
+
+// SyncOneBlock receives a PartialBlock, makes sure it's the next block in the chain, executes it and inserts it to the blockchain.
+func (s *DASyncer) SyncOneBlock(block *da.PartialBlock) error {
+	currentBlock := s.blockchain.CurrentBlock()
+
+	// we expect blocks to be consecutive. block.PartialHeader.Number == parentBlock.Number+1.
+	if block.PartialHeader.Number <= currentBlock.Number().Uint64() {
+		log.Debug("block number is too low", "block number", block.PartialHeader.Number, "parent block number", currentBlock.Number().Uint64())
+		return ErrBlockTooLow
+	} else if block.PartialHeader.Number > currentBlock.Number().Uint64()+1 {
+		log.Debug("block number is too high", "block number", block.PartialHeader.Number, "parent block number", currentBlock.Number().Uint64())
+		return ErrBlockTooHigh
+	}
+
+	parentBlock := s.blockchain.GetBlockByNumber(currentBlock.Number().Uint64())
+	if parentBlock == nil {
+		return fmt.Errorf("parent block not found at height %d", currentBlock.Number().Uint64())
+	}
+
+	if _, err := s.blockchain.BuildAndWriteBlock(parentBlock, block.PartialHeader.ToHeader(), block.Transactions); err != nil {
+		return fmt.Errorf("failed building and writing block, number: %d, error: %v", block.PartialHeader.Number, err)
+	}
+
+	if s.blockchain.CurrentBlock().Number().Uint64()%1000 == 0 {
+		log.Info("L1 sync progress", "blockchain height", s.blockchain.CurrentBlock().Number().Uint64(), "block hash", s.blockchain.CurrentBlock().Hash(), "root", s.blockchain.CurrentBlock().Root())
+	}
+
+	return nil
+}

rollup/da_syncer/data_source.go

@@ -0,0 +1,39 @@
+package da_syncer
+
+import (
+	"context"
+
+	"github.com/scroll-tech/go-ethereum/core"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/params"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/da"
+	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
+)
+
+type DataSource interface {
+	NextData() (da.Entries, error)
+	L1Height() uint64
+}
+
+type DataSourceFactory struct {
+	config        Config
+	genesisConfig *params.ChainConfig
+	l1Client      *rollup_sync_service.L1Client
+	blobClient    blob_client.BlobClient
+	db            ethdb.Database
+}
+
+func NewDataSourceFactory(blockchain *core.BlockChain, genesisConfig *params.ChainConfig, config Config, l1Client *rollup_sync_service.L1Client, blobClient blob_client.BlobClient, db ethdb.Database) *DataSourceFactory {
+	return &DataSourceFactory{
+		config:        config,
+		genesisConfig: genesisConfig,
+		l1Client:      l1Client,
+		blobClient:    blobClient,
+		db:            db,
+	}
+}
+
+func (ds *DataSourceFactory) OpenDataSource(ctx context.Context, l1height uint64) (DataSource, error) {
+	return da.NewCalldataBlobSource(ctx, l1height, ds.l1Client, ds.blobClient, ds.db)
+}

rollup/da_syncer/serrors/errors.go

@@ -0,0 +1,62 @@
+package serrors
+
+import (
+	"fmt"
+)
+
+const (
+	temporary Type = iota
+	eof
+)
+
+var (
+	TemporaryError = NewTemporaryError(nil)
+	EOFError       = NewEOFError(nil)
+)
+
+type Type uint8
+
+func (t Type) String() string {
+	switch t {
+	case temporary:
+		return "temporary"
+	case eof:
+		return "EOF"
+	default:
+		return "unknown"
+	}
+}
+
+type syncError struct {
+	t   Type
+	err error
+}
+
+func NewTemporaryError(err error) error {
+	return &syncError{t: temporary, err: err}
+}
+
+func NewEOFError(err error) error {
+	return &syncError{t: eof, err: err}
+}
+
+func (s *syncError) Error() string {
+	return fmt.Sprintf("%s: %v", s.t, s.err)
+}
+
+func (s *syncError) Unwrap() error {
+	return s.err
+}
+
+func (s *syncError) Is(target error) bool {
+	if target == nil {
+		return s == nil
+	}
+
+	targetSyncErr, ok := target.(*syncError)
+	if !ok {
+		return false
+	}
+
+	return s.t == targetSyncErr.t
+}

rollup/da_syncer/syncing_pipeline.go

@@ -0,0 +1,231 @@
+package da_syncer
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/scroll-tech/go-ethereum/common/backoff"
+	"github.com/scroll-tech/go-ethereum/core"
+	"github.com/scroll-tech/go-ethereum/core/rawdb"
+	"github.com/scroll-tech/go-ethereum/ethdb"
+	"github.com/scroll-tech/go-ethereum/log"
+	"github.com/scroll-tech/go-ethereum/params"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/blob_client"
+	"github.com/scroll-tech/go-ethereum/rollup/da_syncer/serrors"
+	"github.com/scroll-tech/go-ethereum/rollup/rollup_sync_service"
+	"github.com/scroll-tech/go-ethereum/rollup/sync_service"
+)
+
+// Config is the configuration parameters of data availability syncing.
+type Config struct {
+	BlobScanAPIEndpoint    string // BlobScan blob api endpoint
+	BlockNativeAPIEndpoint string // BlockNative blob api endpoint
+	BeaconNodeAPIEndpoint  string // Beacon node api endpoint
+}
+
+// SyncingPipeline is a derivation pipeline for syncing data from L1 and DA and transform it into
+// L2 blocks and chain.
+type SyncingPipeline struct {
+	ctx        context.Context
+	cancel     context.CancelFunc
+	wg         sync.WaitGroup
+	expBackoff *backoff.Exponential
+
+	l1DeploymentBlock uint64
+
+	db         ethdb.Database
+	blockchain *core.BlockChain
+	blockQueue *BlockQueue
+	daSyncer   *DASyncer
+}
+
+func NewSyncingPipeline(ctx context.Context, blockchain *core.BlockChain, genesisConfig *params.ChainConfig, db ethdb.Database, ethClient sync_service.EthClient, l1DeploymentBlock uint64, config Config) (*SyncingPipeline, error) {
+	scrollChainABI, err := rollup_sync_service.ScrollChainMetaData.GetAbi()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get scroll chain abi: %w", err)
+	}
+
+	l1Client, err := rollup_sync_service.NewL1Client(ctx, ethClient, genesisConfig.Scroll.L1Config.L1ChainId, genesisConfig.Scroll.L1Config.ScrollChainAddress, scrollChainABI)
+	if err != nil {
+		return nil, err
+	}
+
+	blobClientList := blob_client.NewBlobClients()
+	if config.BeaconNodeAPIEndpoint != "" {
+		beaconNodeClient, err := blob_client.NewBeaconNodeClient(config.BeaconNodeAPIEndpoint, l1Client)
+		if err != nil {
+			log.Warn("failed to create BeaconNodeClient", "err", err)
+		} else {
+			blobClientList.AddBlobClient(beaconNodeClient)
+		}
+	}
+	if config.BlobScanAPIEndpoint != "" {
+		blobClientList.AddBlobClient(blob_client.NewBlobScanClient(config.BlobScanAPIEndpoint))
+	}
+	if config.BlockNativeAPIEndpoint != "" {
+		blobClientList.AddBlobClient(blob_client.NewBlockNativeClient(config.BlockNativeAPIEndpoint))
+	}
+	if blobClientList.Size() == 0 {
+		return nil, errors.New("DA syncing is enabled but no blob client is configured. Please provide at least one blob client via command line flag")
+	}
+
+	dataSourceFactory := NewDataSourceFactory(blockchain, genesisConfig, config, l1Client, blobClientList, db)
+	syncedL1Height := l1DeploymentBlock - 1
+	from := rawdb.ReadDASyncedL1BlockNumber(db)
+	if from != nil {
+		syncedL1Height = *from
+	}
+
+	daQueue := NewDAQueue(syncedL1Height, dataSourceFactory)
+	batchQueue := NewBatchQueue(daQueue, db)
+	blockQueue := NewBlockQueue(batchQueue)
+	daSyncer := NewDASyncer(blockchain)
+
+	ctx, cancel := context.WithCancel(ctx)
+	return &SyncingPipeline{
+		ctx:               ctx,
+		cancel:            cancel,
+		expBackoff:        backoff.NewExponential(100*time.Millisecond, 10*time.Second, 100*time.Millisecond),
+		wg:                sync.WaitGroup{},
+		l1DeploymentBlock: l1DeploymentBlock,
+		db:                db,
+		blockchain:        blockchain,
+		blockQueue:        blockQueue,
+		daSyncer:          daSyncer,
+	}, nil
+}
+
+func (s *SyncingPipeline) Step() error {
+	block, err := s.blockQueue.NextBlock(s.ctx)
+	if err != nil {
+		return err
+	}
+	err = s.daSyncer.SyncOneBlock(block)
+	return err
+}
+
+func (s *SyncingPipeline) Start() {
+	log.Info("sync from DA: starting pipeline")
+
+	s.wg.Add(1)
+	go func() {
+		s.mainLoop()
+		s.wg.Done()
+	}()
+}
+
+func (s *SyncingPipeline) mainLoop() {
+	stepCh := make(chan struct{}, 1)
+	var delayedStepCh <-chan time.Time
+	var resetCounter int
+	var tempErrorCounter int
+
+	// reqStep is a helper function to request a step to be executed.
+	// If delay is true, it will request a delayed step with exponential backoff, otherwise it will request an immediate step.
+	reqStep := func(delay bool) {
+		if delay {
+			if delayedStepCh == nil {
+				delayDur := s.expBackoff.NextDuration()
+				delayedStepCh = time.After(delayDur)
+				log.Debug("requesting delayed step", "delay", delayDur, "attempt", s.expBackoff.Attempt())
+			} else {
+				log.Debug("ignoring step request because of ongoing delayed step", "attempt", s.expBackoff.Attempt())
+			}
+		} else {
+			select {
+			case stepCh <- struct{}{}:
+			default:
+			}
+		}
+	}
+
+	// start pipeline
+	reqStep(false)
+
+	for {
+		select {
+		case <-s.ctx.Done():
+			return
+		default:
+		}
+
+		select {
+		case <-s.ctx.Done():
+			return
+		case <-delayedStepCh:
+			delayedStepCh = nil
+			reqStep(false)
+		case <-stepCh:
+			err := s.Step()
+			if err == nil {
+				// step succeeded, reset exponential backoff and continue
+				reqStep(false)
+				s.expBackoff.Reset()
+				resetCounter = 0
+				tempErrorCounter = 0
+				continue
+			}
+
+			if errors.Is(err, serrors.EOFError) {
+				// pipeline is empty, request a delayed step
+				// TODO: eventually (with state manager) this should not trigger a delayed step because external events will trigger a new step anyway
+				reqStep(true)
+				tempErrorCounter = 0
+				continue
+			} else if errors.Is(err, serrors.TemporaryError) {
+				log.Warn("syncing pipeline step failed due to temporary error, retrying", "err", err)
+				if tempErrorCounter > 100 {
+					log.Warn("syncing pipeline step failed due to 100 consecutive temporary errors, stopping pipeline worker", "last err", err)
+					return
+				}
+
+				// temporary error, request a delayed step
+				reqStep(true)
+				tempErrorCounter++
+				continue
+			} else if errors.Is(err, ErrBlockTooLow) {
+				// block number returned by the block queue is too low,
+				// we skip the blocks until we reach the correct block number again.
+				reqStep(false)
+				tempErrorCounter = 0
+				continue
+			} else if errors.Is(err, ErrBlockTooHigh) {
+				// block number returned by the block queue is too high,
+				// reset the pipeline and move backwards from the last L1 block we read
+				s.reset(resetCounter)
+				resetCounter++
+				reqStep(false)
+				tempErrorCounter = 0
+				continue
+			} else if errors.Is(err, context.Canceled) {
+				log.Info("syncing pipeline stopped due to cancelled context", "err", err)
+				return
+			}
+
+			log.Warn("syncing pipeline step failed due to unrecoverable error, stopping pipeline worker", "err", err)
+			return
+		}
+	}
+}
+
+func (s *SyncingPipeline) Stop() {
+	log.Info("sync from DA: stopping pipeline...")
+	s.cancel()
+	s.wg.Wait()
+	log.Info("sync from DA: stopping pipeline... done")
+}
+
+func (s *SyncingPipeline) reset(resetCounter int) {
+	amount := 100 * uint64(resetCounter)
+	syncedL1Height := s.l1DeploymentBlock - 1
+	from := rawdb.ReadDASyncedL1BlockNumber(s.db)
+	if from != nil && *from+amount > syncedL1Height {
+		syncedL1Height = *from - amount
+		rawdb.WriteDASyncedL1BlockNumber(s.db, syncedL1Height)
+	}
+	log.Info("resetting syncing pipeline", "syncedL1Height", syncedL1Height)
+	s.blockQueue.Reset(syncedL1Height)
+}

rollup/rollup_sync_service/abi_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestEventSignatures(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	if err != nil {
 		t.Fatal("failed to get scroll chain abi", "err", err)
 	}
@@ -24,7 +24,7 @@ func TestEventSignatures(t *testing.T) {
 }
 
 func TestUnpackLog(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	require.NoError(t, err)
 
 	mockBatchIndex := big.NewInt(123)

rollup/rollup_sync_service/l1client.go

@@ -27,9 +27,9 @@ type L1Client struct {
 	l1FinalizeBatchEventSignature common.Hash
 }
 
-// newL1Client initializes a new L1Client instance with the provided configuration.
+// NewL1Client initializes a new L1Client instance with the provided configuration.
 // It checks for a valid scrollChainAddress and verifies the chain ID.
-func newL1Client(ctx context.Context, l1Client sync_service.EthClient, l1ChainId uint64, scrollChainAddress common.Address, scrollChainABI *abi.ABI) (*L1Client, error) {
+func NewL1Client(ctx context.Context, l1Client sync_service.EthClient, l1ChainId uint64, scrollChainAddress common.Address, scrollChainABI *abi.ABI) (*L1Client, error) {
 	if scrollChainAddress == (common.Address{}) {
 		return nil, errors.New("must pass non-zero scrollChainAddress to L1Client")
 	}
@@ -55,9 +55,9 @@ func newL1Client(ctx context.Context, l1Client sync_service.EthClient, l1ChainId
 	return &client, nil
 }
 
-// fetcRollupEventsInRange retrieves and parses commit/revert/finalize rollup events between block numbers: [from, to].
-func (c *L1Client) fetchRollupEventsInRange(from, to uint64) ([]types.Log, error) {
-	log.Trace("L1Client fetchRollupEventsInRange", "fromBlock", from, "toBlock", to)
+// FetchRollupEventsInRange retrieves and parses commit/revert/finalize rollup events between block numbers: [from, to].
+func (c *L1Client) FetchRollupEventsInRange(from, to uint64) ([]types.Log, error) {
+	log.Trace("L1Client FetchRollupEventsInRange", "fromBlock", from, "toBlock", to)
 
 	query := ethereum.FilterQuery{
 		FromBlock: big.NewInt(int64(from)), // inclusive
@@ -79,8 +79,8 @@ func (c *L1Client) fetchRollupEventsInRange(from, to uint64) ([]types.Log, error
 	return logs, nil
 }
 
-// getLatestFinalizedBlockNumber fetches the block number of the latest finalized block from the L1 chain.
-func (c *L1Client) getLatestFinalizedBlockNumber() (uint64, error) {
+// GetLatestFinalizedBlockNumber fetches the block number of the latest finalized block from the L1 chain.
+func (c *L1Client) GetLatestFinalizedBlockNumber() (uint64, error) {
 	header, err := c.client.HeaderByNumber(c.ctx, big.NewInt(int64(rpc.FinalizedBlockNumber)))
 	if err != nil {
 		return 0, err
@@ -90,3 +90,69 @@ func (c *L1Client) getLatestFinalizedBlockNumber() (uint64, error) {
 	}
 	return header.Number.Uint64(), nil
 }
+
+// FetchTxData fetches tx data corresponding to given event log
+func (c *L1Client) FetchTxData(vLog *types.Log) ([]byte, error) {
+	tx, _, err := c.client.TransactionByHash(c.ctx, vLog.TxHash)
+	if err != nil {
+		log.Debug("failed to get transaction by hash, probably an unindexed transaction, fetching the whole block to get the transaction",
+			"tx hash", vLog.TxHash.Hex(), "block number", vLog.BlockNumber, "block hash", vLog.BlockHash.Hex(), "err", err)
+		block, err := c.client.BlockByHash(c.ctx, vLog.BlockHash)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get block by hash, block number: %v, block hash: %v, err: %w", vLog.BlockNumber, vLog.BlockHash.Hex(), err)
+		}
+
+		found := false
+		for _, txInBlock := range block.Transactions() {
+			if txInBlock.Hash() == vLog.TxHash {
+				tx = txInBlock
+				found = true
+				break
+			}
+		}
+		if !found {
+			return nil, fmt.Errorf("transaction not found in the block, tx hash: %v, block number: %v, block hash: %v", vLog.TxHash.Hex(), vLog.BlockNumber, vLog.BlockHash.Hex())
+		}
+	}
+
+	return tx.Data(), nil
+}
+
+// FetchTxBlobHash fetches tx blob hash corresponding to given event log
+func (c *L1Client) FetchTxBlobHash(vLog *types.Log) (common.Hash, error) {
+	tx, _, err := c.client.TransactionByHash(c.ctx, vLog.TxHash)
+	if err != nil {
+		log.Debug("failed to get transaction by hash, probably an unindexed transaction, fetching the whole block to get the transaction",
+			"tx hash", vLog.TxHash.Hex(), "block number", vLog.BlockNumber, "block hash", vLog.BlockHash.Hex(), "err", err)
+		block, err := c.client.BlockByHash(c.ctx, vLog.BlockHash)
+		if err != nil {
+			return common.Hash{}, fmt.Errorf("failed to get block by hash, block number: %v, block hash: %v, err: %w", vLog.BlockNumber, vLog.BlockHash.Hex(), err)
+		}
+
+		found := false
+		for _, txInBlock := range block.Transactions() {
+			if txInBlock.Hash() == vLog.TxHash {
+				tx = txInBlock
+				found = true
+				break
+			}
+		}
+		if !found {
+			return common.Hash{}, fmt.Errorf("transaction not found in the block, tx hash: %v, block number: %v, block hash: %v", vLog.TxHash.Hex(), vLog.BlockNumber, vLog.BlockHash.Hex())
+		}
+	}
+	blobHashes := tx.BlobHashes()
+	if len(blobHashes) == 0 {
+		return common.Hash{}, fmt.Errorf("transaction does not contain any blobs, tx hash: %v", vLog.TxHash.Hex())
+	}
+	return blobHashes[0], nil
+}
+
+// GetHeaderByNumber fetches the block header by number
+func (c *L1Client) GetHeaderByNumber(blockNumber uint64) (*types.Header, error) {
+	header, err := c.client.HeaderByNumber(c.ctx, big.NewInt(0).SetUint64(blockNumber))
+	if err != nil {
+		return nil, err
+	}
+	return header, nil
+}

rollup/rollup_sync_service/l1client_test.go

@@ -18,21 +18,21 @@ func TestL1Client(t *testing.T) {
 	ctx := context.Background()
 	mockClient := &mockEthClient{}
 
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	if err != nil {
 		t.Fatal("failed to get scroll chain abi", "err", err)
 	}
 	scrollChainAddress := common.HexToAddress("0x0123456789abcdef")
-	l1Client, err := newL1Client(ctx, mockClient, 11155111, scrollChainAddress, scrollChainABI)
+	l1Client, err := NewL1Client(ctx, mockClient, 11155111, scrollChainAddress, scrollChainABI)
 	require.NoError(t, err, "Failed to initialize L1Client")
 
-	blockNumber, err := l1Client.getLatestFinalizedBlockNumber()
+	blockNumber, err := l1Client.GetLatestFinalizedBlockNumber()
 	assert.NoError(t, err, "Error getting latest confirmed block number")
 	assert.Equal(t, uint64(36), blockNumber, "Unexpected block number")
 
-	logs, err := l1Client.fetchRollupEventsInRange(0, blockNumber)
+	logs, err := l1Client.FetchRollupEventsInRange(0, blockNumber)
 	assert.NoError(t, err, "Error fetching rollup events in range")
-	assert.Empty(t, logs, "Expected no logs from fetchRollupEventsInRange")
+	assert.Empty(t, logs, "Expected no logs from FetchRollupEventsInRange")
 }
 
 type mockEthClient struct {

rollup/rollup_sync_service/rollup_sync_service.go

@@ -4,18 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"math/big"
 	"os"
 	"reflect"
 	"sync"
 	"time"
 
 	"github.com/scroll-tech/da-codec/encoding"
-	"github.com/scroll-tech/da-codec/encoding/codecv0"
-	"github.com/scroll-tech/da-codec/encoding/codecv1"
-	"github.com/scroll-tech/da-codec/encoding/codecv2"
-	"github.com/scroll-tech/da-codec/encoding/codecv3"
-	"github.com/scroll-tech/da-codec/encoding/codecv4"
 
 	"github.com/scroll-tech/go-ethereum/accounts/abi"
 	"github.com/scroll-tech/go-ethereum/common"
@@ -26,7 +20,6 @@ import (
 	"github.com/scroll-tech/go-ethereum/log"
 	"github.com/scroll-tech/go-ethereum/node"
 	"github.com/scroll-tech/go-ethereum/params"
-
 	"github.com/scroll-tech/go-ethereum/rollup/rcfg"
 	"github.com/scroll-tech/go-ethereum/rollup/sync_service"
 	"github.com/scroll-tech/go-ethereum/rollup/withdrawtrie"
@@ -78,12 +71,12 @@ func NewRollupSyncService(ctx context.Context, genesisConfig *params.ChainConfig
 		return nil, fmt.Errorf("missing L1 config in genesis")
 	}
 
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get scroll chain abi: %w", err)
 	}
 
-	client, err := newL1Client(ctx, l1Client, genesisConfig.Scroll.L1Config.L1ChainId, genesisConfig.Scroll.L1Config.ScrollChainAddress, scrollChainABI)
+	client, err := NewL1Client(ctx, l1Client, genesisConfig.Scroll.L1Config.L1ChainId, genesisConfig.Scroll.L1Config.ScrollChainAddress, scrollChainABI)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize l1 client: %w", err)
 	}
@@ -176,7 +169,7 @@ func (s *RollupSyncService) fetchRollupEvents() {
 	s.stateMu.Lock()
 	defer s.stateMu.Unlock()
 
-	latestConfirmed, err := s.client.getLatestFinalizedBlockNumber()
+	latestConfirmed, err := s.client.GetLatestFinalizedBlockNumber()
 	if err != nil {
 		log.Warn("failed to get latest confirmed block number", "err", err)
 		return
@@ -196,7 +189,7 @@ func (s *RollupSyncService) fetchRollupEvents() {
 			to = latestConfirmed
 		}
 
-		logs, err := s.client.fetchRollupEventsInRange(from, to)
+		logs, err := s.client.FetchRollupEventsInRange(from, to)
 		if err != nil {
 			log.Error("failed to fetch rollup events in range", "from block", from, "to block", to, "err", err)
 			return
@@ -222,12 +215,11 @@ func (s *RollupSyncService) parseAndUpdateRollupEventLogs(logs []types.Log, endB
 			batchIndex := event.BatchIndex.Uint64()
 			log.Trace("found new CommitBatch event", "batch index", batchIndex)
 
-			committedBatchMeta, chunkBlockRanges, err := s.getCommittedBatchMeta(batchIndex, &vLog)
+			committedBatchMeta, err := s.getCommittedBatchMeta(batchIndex, &vLog)
 			if err != nil {
 				return fmt.Errorf("failed to get chunk ranges, batch index: %v, err: %w", batchIndex, err)
 			}
 			rawdb.WriteCommittedBatchMeta(s.db, batchIndex, committedBatchMeta)
-			rawdb.WriteBatchChunkRanges(s.db, batchIndex, chunkBlockRanges)
 
 		case s.l1RevertBatchEventSignature:
 			event := &L1RevertBatchEvent{}
@@ -238,7 +230,6 @@ func (s *RollupSyncService) parseAndUpdateRollupEventLogs(logs []types.Log, endB
 			log.Trace("found new RevertBatch event", "batch index", batchIndex)
 
 			rawdb.DeleteCommittedBatchMeta(s.db, batchIndex)
-			rawdb.DeleteBatchChunkRanges(s.db, batchIndex)
 
 		case s.l1FinalizeBatchEventSignature:
 			event := &L1FinalizeBatchEvent{}
@@ -273,12 +264,12 @@ func (s *RollupSyncService) parseAndUpdateRollupEventLogs(logs []types.Log, endB
 			for index := startBatchIndex; index <= batchIndex; index++ {
 				committedBatchMeta := rawdb.ReadCommittedBatchMeta(s.db, index)
 
-				chunks, err := s.getLocalChunksForBatch(index)
+				chunks, err := s.getLocalChunksForBatch(committedBatchMeta.ChunkBlockRanges)
 				if err != nil {
 					return fmt.Errorf("failed to get local node info, batch index: %v, err: %w", index, err)
 				}
 
-				endBlock, finalizedBatchMeta, err := validateBatch(index, event, parentFinalizedBatchMeta, committedBatchMeta, chunks, s.bc.Config(), s.stack)
+				endBlock, finalizedBatchMeta, err := validateBatch(index, event, parentFinalizedBatchMeta, committedBatchMeta, chunks, s.stack)
 				if err != nil {
 					return fmt.Errorf("fatal: validateBatch failed: finalize event: %v, err: %w", event, err)
 				}
@@ -313,12 +304,10 @@ func (s *RollupSyncService) parseAndUpdateRollupEventLogs(logs []types.Log, endB
 	return nil
 }
 
-func (s *RollupSyncService) getLocalChunksForBatch(batchIndex uint64) ([]*encoding.Chunk, error) {
-	chunkBlockRanges := rawdb.ReadBatchChunkRanges(s.db, batchIndex)
+func (s *RollupSyncService) getLocalChunksForBatch(chunkBlockRanges []*rawdb.ChunkBlockRange) ([]*encoding.Chunk, error) {
 	if len(chunkBlockRanges) == 0 {
-		return nil, fmt.Errorf("failed to get batch chunk ranges, empty chunk block ranges")
+		return nil, fmt.Errorf("chunkBlockRanges is empty")
 	}
-
 	endBlockNumber := chunkBlockRanges[len(chunkBlockRanges)-1].EndBlockNumber
 	for i := 0; i < defaultMaxRetries; i++ {
 		if s.ctx.Err() != nil {
@@ -366,13 +355,13 @@ func (s *RollupSyncService) getLocalChunksForBatch(batchIndex uint64) ([]*encodi
 	return chunks, nil
 }
 
-func (s *RollupSyncService) getCommittedBatchMeta(batchIndex uint64, vLog *types.Log) (*rawdb.CommittedBatchMeta, []*rawdb.ChunkBlockRange, error) {
+func (s *RollupSyncService) getCommittedBatchMeta(batchIndex uint64, vLog *types.Log) (*rawdb.CommittedBatchMeta, error) {
 	if batchIndex == 0 {
 		return &rawdb.CommittedBatchMeta{
 			Version:             0,
 			BlobVersionedHashes: nil,
 			ChunkBlockRanges:    []*rawdb.ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 0}},
-		}, []*rawdb.ChunkBlockRange{{StartBlockNumber: 0, EndBlockNumber: 0}}, nil
+		}, nil
 	}
 
 	tx, _, err := s.client.client.TransactionByHash(s.ctx, vLog.TxHash)
@@ -381,11 +370,11 @@ func (s *RollupSyncService) getCommittedBatchMeta(batchIndex uint64, vLog *types
 			"tx hash", vLog.TxHash.Hex(), "block number", vLog.BlockNumber, "block hash", vLog.BlockHash.Hex(), "err", err)
 		block, err := s.client.client.BlockByHash(s.ctx, vLog.BlockHash)
 		if err != nil {
-			return nil, nil, fmt.Errorf("failed to get block by hash, block number: %v, block hash: %v, err: %w", vLog.BlockNumber, vLog.BlockHash.Hex(), err)
+			return nil, fmt.Errorf("failed to get block by hash, block number: %v, block hash: %v, err: %w", vLog.BlockNumber, vLog.BlockHash.Hex(), err)
 		}
 
 		if block == nil {
-			return nil, nil, fmt.Errorf("failed to get block by hash, block not found, block number: %v, block hash: %v", vLog.BlockNumber, vLog.BlockHash.Hex())
+			return nil, fmt.Errorf("failed to get block by hash, block not found, block number: %v, block hash: %v", vLog.BlockNumber, vLog.BlockHash.Hex())
 		}
 
 		found := false
@@ -397,7 +386,7 @@ func (s *RollupSyncService) getCommittedBatchMeta(batchIndex uint64, vLog *types
 			}
 		}
 		if !found {
-			return nil, nil, fmt.Errorf("transaction not found in the block, tx hash: %v, block number: %v, block hash: %v", vLog.TxHash.Hex(), vLog.BlockNumber, vLog.BlockHash.Hex())
+			return nil, fmt.Errorf("transaction not found in the block, tx hash: %v, block number: %v, block hash: %v", vLog.TxHash.Hex(), vLog.BlockNumber, vLog.BlockHash.Hex())
 		}
 	}
 
@@ -406,19 +395,19 @@ func (s *RollupSyncService) getCommittedBatchMeta(batchIndex uint64, vLog *types
 	if tx.Type() == types.BlobTxType {
 		blobVersionedHashes := tx.BlobHashes()
 		if blobVersionedHashes == nil {
-			return nil, nil, fmt.Errorf("invalid blob transaction, blob hashes is nil, tx hash: %v", tx.Hash().Hex())
+			return nil, fmt.Errorf("invalid blob transaction, blob hashes is nil, tx hash: %v", tx.Hash().Hex())
 		}
 		commitBatchMeta.BlobVersionedHashes = blobVersionedHashes
 	}
 
 	version, ranges, err := s.decodeBatchVersionAndChunkBlockRanges(tx.Data())
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to decode chunk block ranges, batch index: %v, err: %w", batchIndex, err)
+		return nil, fmt.Errorf("failed to decode chunk block ranges, batch index: %v, err: %w", batchIndex, err)
 	}
 
 	commitBatchMeta.Version = version
 	commitBatchMeta.ChunkBlockRanges = ranges
-	return &commitBatchMeta, ranges, nil
+	return &commitBatchMeta, nil
 }
 
 // decodeBatchVersionAndChunkBlockRanges decodes version and chunks' block ranges in a batch based on the commit batch transaction's calldata.
@@ -493,10 +482,8 @@ func (s *RollupSyncService) decodeBatchVersionAndChunkBlockRanges(txData []byte)
 //   - batchIndex: batch index of the validated batch
 //   - event: L1 finalize batch event data
 //   - parentFinalizedBatchMeta: metadata of the finalized parent batch
-//   - committedBatchMeta: committed batch metadata stored in the database.
-//     Can be nil for older client versions that don't store this information.
+//   - committedBatchMeta: committed batch metadata stored in the database
 //   - chunks: slice of chunk data for the current batch
-//   - chainCfg: chain configuration to identify the codec version when committedBatchMeta is nil
 //   - stack: node stack to terminate the node in case of inconsistency
 //
 // Returns:
@@ -507,7 +494,7 @@ func (s *RollupSyncService) decodeBatchVersionAndChunkBlockRanges(txData []byte)
 // Note: This function is compatible with both "finalize by batch" and "finalize by bundle" methods.
 // In "finalize by bundle", only the last batch of each bundle is fully verified.
 // This check still ensures the correctness of all batch hashes in the bundle due to the parent-child relationship between batch hashes.
-func validateBatch(batchIndex uint64, event *L1FinalizeBatchEvent, parentFinalizedBatchMeta *rawdb.FinalizedBatchMeta, committedBatchMeta *rawdb.CommittedBatchMeta, chunks []*encoding.Chunk, chainCfg *params.ChainConfig, stack *node.Node) (uint64, *rawdb.FinalizedBatchMeta, error) {
+func validateBatch(batchIndex uint64, event *L1FinalizeBatchEvent, parentFinalizedBatchMeta *rawdb.FinalizedBatchMeta, committedBatchMeta *rawdb.CommittedBatchMeta, chunks []*encoding.Chunk, stack *node.Node) (uint64, *rawdb.FinalizedBatchMeta, error) {
 	if len(chunks) == 0 {
 		return 0, nil, fmt.Errorf("invalid argument: length of chunks is 0, batch index: %v", batchIndex)
 	}
@@ -532,71 +519,17 @@ func validateBatch(batchIndex uint64, event *L1FinalizeBatchEvent, parentFinaliz
 		Chunks:                     chunks,
 	}
 
-	var codecVersion encoding.CodecVersion
-	if committedBatchMeta != nil {
-		codecVersion = encoding.CodecVersion(committedBatchMeta.Version)
-	} else {
-		codecVersion = determineCodecVersion(startBlock.Header.Number, startBlock.Header.Time, chainCfg)
+	codecVersion := encoding.CodecVersion(committedBatchMeta.Version)
+	codec, err := encoding.CodecFromVersion(codecVersion)
+	if err != nil {
+		return 0, nil, fmt.Errorf("unsupported codec version: %v, batch index: %v, err: %w", codecVersion, batchIndex, err)
 	}
 
-	var localBatchHash common.Hash
-	if codecVersion == encoding.CodecV0 {
-		daBatch, err := codecv0.NewDABatch(batch)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to create codecv0 DA batch, batch index: %v, err: %w", batchIndex, err)
-		}
-		localBatchHash = daBatch.Hash()
-	} else if codecVersion == encoding.CodecV1 {
-		daBatch, err := codecv1.NewDABatch(batch)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to create codecv1 DA batch, batch index: %v, err: %w", batchIndex, err)
-		}
-		localBatchHash = daBatch.Hash()
-	} else if codecVersion == encoding.CodecV2 {
-		daBatch, err := codecv2.NewDABatch(batch)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to create codecv2 DA batch, batch index: %v, err: %w", batchIndex, err)
-		}
-		localBatchHash = daBatch.Hash()
-	} else if codecVersion == encoding.CodecV3 {
-		daBatch, err := codecv3.NewDABatch(batch)
-		if err != nil {
-			return 0, nil, fmt.Errorf("failed to create codecv3 DA batch, batch index: %v, err: %w", batchIndex, err)
-		}
-		localBatchHash = daBatch.Hash()
-	} else if codecVersion == encoding.CodecV4 {
-		// Check if committedBatchMeta exists, for backward compatibility with older client versions
-		if committedBatchMeta == nil {
-			return 0, nil, fmt.Errorf("missing committed batch metadata for codecV4, please use the latest client version, batch index: %v", batchIndex)
-		}
-
-		// Validate BlobVersionedHashes
-		if committedBatchMeta.BlobVersionedHashes == nil || len(committedBatchMeta.BlobVersionedHashes) != 1 {
-			return 0, nil, fmt.Errorf("invalid blob hashes, batch index: %v, blob hashes: %v", batchIndex, committedBatchMeta.BlobVersionedHashes)
-		}
-
-		// Attempt to create DA batch with compression
-		daBatch, err := codecv4.NewDABatch(batch, true)
-		if err != nil {
-			// If compression fails, try without compression
-			log.Warn("failed to create codecv4 DA batch with compress enabling", "batch index", batchIndex, "err", err)
-			daBatch, err = codecv4.NewDABatch(batch, false)
-			if err != nil {
-				return 0, nil, fmt.Errorf("failed to create codecv4 DA batch, batch index: %v, err: %w", batchIndex, err)
-			}
-		} else if daBatch.BlobVersionedHash != committedBatchMeta.BlobVersionedHashes[0] {
-			// Inconsistent blob versioned hash, fallback to uncompressed DA batch
-			log.Warn("impossible case: inconsistent blob versioned hash", "batch index", batchIndex, "expected", committedBatchMeta.BlobVersionedHashes[0], "actual", daBatch.BlobVersionedHash)
-			daBatch, err = codecv4.NewDABatch(batch, false)
-			if err != nil {
-				return 0, nil, fmt.Errorf("failed to create codecv4 DA batch, batch index: %v, err: %w", batchIndex, err)
-			}
-		}
-
-		localBatchHash = daBatch.Hash()
-	} else {
-		return 0, nil, fmt.Errorf("unsupported codec version: %v", codecVersion)
+	daBatch, err := codec.NewDABatch(batch)
+	if err != nil {
+		return 0, nil, fmt.Errorf("failed to create DA batch, batch index: %v, codec version: %v, expected blob hashes: %v, err: %w", batchIndex, codecVersion, committedBatchMeta.BlobVersionedHashes, err)
 	}
+	localBatchHash := daBatch.Hash()
 
 	localStateRoot := endBlock.Header.Root
 	localWithdrawRoot := endBlock.WithdrawRoot
@@ -648,126 +581,29 @@ func validateBatch(batchIndex uint64, event *L1FinalizeBatchEvent, parentFinaliz
 	return endBlock.Header.Number.Uint64(), finalizedBatchMeta, nil
 }
 
-// determineCodecVersion determines the codec version based on the block number and chain configuration.
-func determineCodecVersion(startBlockNumber *big.Int, startBlockTimestamp uint64, chainCfg *params.ChainConfig) encoding.CodecVersion {
-	switch {
-	case startBlockNumber.Uint64() == 0 || !chainCfg.IsBernoulli(startBlockNumber):
-		return encoding.CodecV0 // codecv0: genesis batch or batches before Bernoulli
-	case !chainCfg.IsCurie(startBlockNumber):
-		return encoding.CodecV1 // codecv1: batches after Bernoulli and before Curie
-	case !chainCfg.IsDarwin(startBlockTimestamp):
-		return encoding.CodecV2 // codecv2: batches after Curie and before Darwin
-	case !chainCfg.IsDarwinV2(startBlockTimestamp):
-		return encoding.CodecV3 // codecv3: batches after Darwin
-	default:
-		return encoding.CodecV4 // codecv4: batches after DarwinV2
-	}
-}
-
 // decodeBlockRangesFromEncodedChunks decodes the provided chunks into a list of block ranges.
 func decodeBlockRangesFromEncodedChunks(codecVersion encoding.CodecVersion, chunks [][]byte) ([]*rawdb.ChunkBlockRange, error) {
-	var chunkBlockRanges []*rawdb.ChunkBlockRange
-	for _, chunk := range chunks {
-		if len(chunk) < 1 {
-			return nil, fmt.Errorf("invalid chunk, length is less than 1")
-		}
-
-		numBlocks := int(chunk[0])
-
-		switch codecVersion {
-		case encoding.CodecV0:
-			if len(chunk) < 1+numBlocks*60 {
-				return nil, fmt.Errorf("invalid chunk byte length, expected: %v, got: %v", 1+numBlocks*60, len(chunk))
-			}
-			daBlocks := make([]*codecv0.DABlock, numBlocks)
-			for i := 0; i < numBlocks; i++ {
-				startIdx := 1 + i*60 // add 1 to skip numBlocks byte
-				endIdx := startIdx + 60
-				daBlocks[i] = &codecv0.DABlock{}
-				if err := daBlocks[i].Decode(chunk[startIdx:endIdx]); err != nil {
-					return nil, err
-				}
-			}
-
-			chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
-				StartBlockNumber: daBlocks[0].BlockNumber,
-				EndBlockNumber:   daBlocks[len(daBlocks)-1].BlockNumber,
-			})
-		case encoding.CodecV1:
-			if len(chunk) != 1+numBlocks*60 {
-				return nil, fmt.Errorf("invalid chunk byte length, expected: %v, got: %v", 1+numBlocks*60, len(chunk))
-			}
-			daBlocks := make([]*codecv1.DABlock, numBlocks)
-			for i := 0; i < numBlocks; i++ {
-				startIdx := 1 + i*60 // add 1 to skip numBlocks byte
-				endIdx := startIdx + 60
-				daBlocks[i] = &codecv1.DABlock{}
-				if err := daBlocks[i].Decode(chunk[startIdx:endIdx]); err != nil {
-					return nil, err
-				}
-			}
-
-			chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
-				StartBlockNumber: daBlocks[0].BlockNumber,
-				EndBlockNumber:   daBlocks[len(daBlocks)-1].BlockNumber,
-			})
-		case encoding.CodecV2:
-			if len(chunk) != 1+numBlocks*60 {
-				return nil, fmt.Errorf("invalid chunk byte length, expected: %v, got: %v", 1+numBlocks*60, len(chunk))
-			}
-			daBlocks := make([]*codecv2.DABlock, numBlocks)
-			for i := 0; i < numBlocks; i++ {
-				startIdx := 1 + i*60 // add 1 to skip numBlocks byte
-				endIdx := startIdx + 60
-				daBlocks[i] = &codecv2.DABlock{}
-				if err := daBlocks[i].Decode(chunk[startIdx:endIdx]); err != nil {
-					return nil, err
-				}
-			}
-
-			chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
-				StartBlockNumber: daBlocks[0].BlockNumber,
-				EndBlockNumber:   daBlocks[len(daBlocks)-1].BlockNumber,
-			})
-		case encoding.CodecV3:
-			if len(chunk) != 1+numBlocks*60 {
-				return nil, fmt.Errorf("invalid chunk byte length, expected: %v, got: %v", 1+numBlocks*60, len(chunk))
-			}
-			daBlocks := make([]*codecv3.DABlock, numBlocks)
-			for i := 0; i < numBlocks; i++ {
-				startIdx := 1 + i*60 // add 1 to skip numBlocks byte
-				endIdx := startIdx + 60
-				daBlocks[i] = &codecv3.DABlock{}
-				if err := daBlocks[i].Decode(chunk[startIdx:endIdx]); err != nil {
-					return nil, err
-				}
-			}
-
-			chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
-				StartBlockNumber: daBlocks[0].BlockNumber,
-				EndBlockNumber:   daBlocks[len(daBlocks)-1].BlockNumber,
-			})
-		case encoding.CodecV4:
-			if len(chunk) != 1+numBlocks*60 {
-				return nil, fmt.Errorf("invalid chunk byte length, expected: %v, got: %v", 1+numBlocks*60, len(chunk))
-			}
-			daBlocks := make([]*codecv4.DABlock, numBlocks)
-			for i := 0; i < numBlocks; i++ {
-				startIdx := 1 + i*60 // add 1 to skip numBlocks byte
-				endIdx := startIdx + 60
-				daBlocks[i] = &codecv4.DABlock{}
-				if err := daBlocks[i].Decode(chunk[startIdx:endIdx]); err != nil {
-					return nil, err
-				}
-			}
-
-			chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
-				StartBlockNumber: daBlocks[0].BlockNumber,
-				EndBlockNumber:   daBlocks[len(daBlocks)-1].BlockNumber,
-			})
-		default:
-			return nil, fmt.Errorf("unexpected batch version %v", codecVersion)
-		}
+	codec, err := encoding.CodecFromVersion(codecVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get codec from version: %v, err: %w", codecVersion, err)
 	}
+
+	daChunksRawTx, err := codec.DecodeDAChunksRawTx(chunks)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode DA chunks, version: %v, err: %w", codecVersion, err)
+	}
+
+	var chunkBlockRanges []*rawdb.ChunkBlockRange
+	for _, daChunkRawTx := range daChunksRawTx {
+		if len(daChunkRawTx.Blocks) == 0 {
+			return nil, fmt.Errorf("no blocks found in DA chunk, version: %v", codecVersion)
+		}
+
+		chunkBlockRanges = append(chunkBlockRanges, &rawdb.ChunkBlockRange{
+			StartBlockNumber: daChunkRawTx.Blocks[0].Number(),
+			EndBlockNumber:   daChunkRawTx.Blocks[len(daChunkRawTx.Blocks)-1].Number(),
+		})
+	}
+
 	return chunkBlockRanges, nil
 }

rollup/rollup_sync_service/rollup_sync_service_test.go

@@ -51,7 +51,7 @@ func TestRollupSyncServiceStartAndStop(t *testing.T) {
 }
 
 func TestDecodeBatchVersionAndChunkBlockRangesCodecv0(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	require.NoError(t, err)
 
 	service := &RollupSyncService{
@@ -110,7 +110,7 @@ func TestDecodeBatchVersionAndChunkBlockRangesCodecv0(t *testing.T) {
 }
 
 func TestDecodeBatchVersionAndChunkBlockRangesCodecv1(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	require.NoError(t, err)
 
 	service := &RollupSyncService{
@@ -163,7 +163,7 @@ func TestDecodeBatchVersionAndChunkBlockRangesCodecv1(t *testing.T) {
 }
 
 func TestDecodeBatchVersionAndChunkBlockRangesCodecv2(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	require.NoError(t, err)
 
 	service := &RollupSyncService{
@@ -216,7 +216,7 @@ func TestDecodeBatchVersionAndChunkBlockRangesCodecv2(t *testing.T) {
 }
 
 func TestDecodeBatchVersionAndChunkBlockRangesCodecv3(t *testing.T) {
-	scrollChainABI, err := scrollChainMetaData.GetAbi()
+	scrollChainABI, err := ScrollChainMetaData.GetAbi()
 	require.NoError(t, err)
 
 	service := &RollupSyncService{
@@ -313,7 +313,7 @@ func TestGetCommittedBatchMetaCodecv0(t *testing.T) {
 	vLog := &types.Log{
 		TxHash: common.HexToHash("0x0"),
 	}
-	metadata, ranges, err := service.getCommittedBatchMeta(1, vLog)
+	metadata, err := service.getCommittedBatchMeta(1, vLog)
 	require.NoError(t, err)
 
 	assert.Equal(t, encoding.CodecV0, encoding.CodecVersion(metadata.Version))
@@ -324,13 +324,13 @@ func TestGetCommittedBatchMetaCodecv0(t *testing.T) {
 		{StartBlockNumber: 911156, EndBlockNumber: 911159},
 	}
 
-	if len(expectedRanges) != len(ranges) {
-		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(ranges))
+	if len(expectedRanges) != len(metadata.ChunkBlockRanges) {
+		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(metadata.ChunkBlockRanges))
 	}
 
-	for i := range ranges {
-		if *expectedRanges[i] != *ranges[i] {
-			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *ranges[i])
+	for i := range metadata.ChunkBlockRanges {
+		if *expectedRanges[i] != *metadata.ChunkBlockRanges[i] {
+			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *metadata.ChunkBlockRanges[i])
 		}
 	}
 }
@@ -367,7 +367,7 @@ func TestGetCommittedBatchMetaCodecv1(t *testing.T) {
 	vLog := &types.Log{
 		TxHash: common.HexToHash("0x1"),
 	}
-	metadata, ranges, err := service.getCommittedBatchMeta(1, vLog)
+	metadata, err := service.getCommittedBatchMeta(1, vLog)
 	require.NoError(t, err)
 
 	assert.Equal(t, encoding.CodecV1, encoding.CodecVersion(metadata.Version))
@@ -376,13 +376,13 @@ func TestGetCommittedBatchMetaCodecv1(t *testing.T) {
 		{StartBlockNumber: 1, EndBlockNumber: 11},
 	}
 
-	if len(expectedRanges) != len(ranges) {
-		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(ranges))
+	if len(expectedRanges) != len(metadata.ChunkBlockRanges) {
+		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(metadata.ChunkBlockRanges))
 	}
 
-	for i := range ranges {
-		if *expectedRanges[i] != *ranges[i] {
-			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *ranges[i])
+	for i := range metadata.ChunkBlockRanges {
+		if *expectedRanges[i] != *metadata.ChunkBlockRanges[i] {
+			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *metadata.ChunkBlockRanges[i])
 		}
 	}
 }
@@ -419,7 +419,7 @@ func TestGetCommittedBatchMetaCodecv2(t *testing.T) {
 	vLog := &types.Log{
 		TxHash: common.HexToHash("0x2"),
 	}
-	metadata, ranges, err := service.getCommittedBatchMeta(1, vLog)
+	metadata, err := service.getCommittedBatchMeta(1, vLog)
 	require.NoError(t, err)
 
 	assert.Equal(t, encoding.CodecV2, encoding.CodecVersion(metadata.Version))
@@ -456,13 +456,13 @@ func TestGetCommittedBatchMetaCodecv2(t *testing.T) {
 		{StartBlockNumber: 174, EndBlockNumber: 174},
 	}
 
-	if len(expectedRanges) != len(ranges) {
-		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(ranges))
+	if len(expectedRanges) != len(metadata.ChunkBlockRanges) {
+		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(metadata.ChunkBlockRanges))
 	}
 
-	for i := range ranges {
-		if *expectedRanges[i] != *ranges[i] {
-			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *ranges[i])
+	for i := range metadata.ChunkBlockRanges {
+		if *expectedRanges[i] != *metadata.ChunkBlockRanges[i] {
+			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *metadata.ChunkBlockRanges[i])
 		}
 	}
 }
@@ -499,7 +499,7 @@ func TestGetCommittedBatchMetaCodecv3(t *testing.T) {
 	vLog := &types.Log{
 		TxHash: common.HexToHash("0x3"),
 	}
-	metadata, ranges, err := service.getCommittedBatchMeta(1, vLog)
+	metadata, err := service.getCommittedBatchMeta(1, vLog)
 	require.NoError(t, err)
 
 	assert.Equal(t, encoding.CodecV3, encoding.CodecVersion(metadata.Version))
@@ -537,20 +537,18 @@ func TestGetCommittedBatchMetaCodecv3(t *testing.T) {
 		{StartBlockNumber: 70, EndBlockNumber: 70},
 	}
 
-	if len(expectedRanges) != len(ranges) {
-		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(ranges))
+	if len(expectedRanges) != len(metadata.ChunkBlockRanges) {
+		t.Fatalf("Expected range length %v, got %v", len(expectedRanges), len(metadata.ChunkBlockRanges))
 	}
 
-	for i := range ranges {
-		if *expectedRanges[i] != *ranges[i] {
-			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *ranges[i])
+	for i := range metadata.ChunkBlockRanges {
+		if *expectedRanges[i] != *metadata.ChunkBlockRanges[i] {
+			t.Fatalf("Mismatch at index %d: expected %v, got %v", i, *expectedRanges[i], *metadata.ChunkBlockRanges[i])
 		}
 	}
 }
 
 func TestValidateBatchCodecv0(t *testing.T) {
-	chainConfig := &params.ChainConfig{}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{block1}}
 
@@ -560,50 +558,57 @@ func TestValidateBatchCodecv0(t *testing.T) {
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
 	chunk3 := &encoding.Chunk{Blocks: []*encoding.Block{block3}}
 
-	parentBatchMeta1 := &rawdb.FinalizedBatchMeta{}
+	parentFinalizedBatchMeta1 := &rawdb.FinalizedBatchMeta{}
 	event1 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(0),
 		BatchHash:    common.HexToHash("0xfd3ecf106ce993adc6db68e42ce701bfe638434395abdeeb871f7bd395ae2368"),
 		StateRoot:    chunk3.Blocks[len(chunk3.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk3.Blocks[len(chunk3.Blocks)-1].WithdrawRoot,
 	}
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV0),
+		BlobVersionedHashes: nil,
+	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentBatchMeta1, nil, []*encoding.Chunk{chunk1, chunk2, chunk3}, chainConfig, nil)
+	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentFinalizedBatchMeta1, committedBatchMeta1, []*encoding.Chunk{chunk1, chunk2, chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock1)
 
 	block4 := readBlockFromJSON(t, "./testdata/blockTrace_05.json")
 	chunk4 := &encoding.Chunk{Blocks: []*encoding.Block{block4}}
 
-	parentBatchMeta2 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta2 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event1.BatchHash,
 		TotalL1MessagePopped: 11,
 		StateRoot:            event1.StateRoot,
 		WithdrawRoot:         event1.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta2, finalizedBatchMeta1)
+	assert.Equal(t, parentFinalizedBatchMeta2, finalizedBatchMeta1)
 	event2 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(1),
 		BatchHash:    common.HexToHash("0xadb8e526c3fdc2045614158300789cd66e7a945efe5a484db00b5ef9a26016d7"),
 		StateRoot:    chunk4.Blocks[len(chunk4.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
-	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentBatchMeta2, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV0),
+		BlobVersionedHashes: nil,
+	}
+
+	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentFinalizedBatchMeta2, committedBatchMeta2, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock2)
 
-	parentBatchMeta3 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta3 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event2.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event2.StateRoot,
 		WithdrawRoot:         event2.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta3, finalizedBatchMeta2)
+	assert.Equal(t, parentFinalizedBatchMeta3, finalizedBatchMeta2)
 }
 
 func TestValidateBatchCodecv1(t *testing.T) {
-	chainConfig := &params.ChainConfig{BernoulliBlock: big.NewInt(0)}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{block1}}
 
@@ -613,50 +618,56 @@ func TestValidateBatchCodecv1(t *testing.T) {
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
 	chunk3 := &encoding.Chunk{Blocks: []*encoding.Block{block3}}
 
-	parentBatchMeta1 := &rawdb.FinalizedBatchMeta{}
+	parentFinalizedBatchMeta1 := &rawdb.FinalizedBatchMeta{}
 	event1 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(0),
 		BatchHash:    common.HexToHash("0x73cb3310646716cb782702a0ec4ad33cf55633c85daf96b641953c5defe58031"),
 		StateRoot:    chunk3.Blocks[len(chunk3.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk3.Blocks[len(chunk3.Blocks)-1].WithdrawRoot,
 	}
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV1),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x0129554070e4323800ca0e5ddd17bc447854601b306a70870002a058741214b3")},
+	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentBatchMeta1, nil, []*encoding.Chunk{chunk1, chunk2, chunk3}, chainConfig, nil)
+	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentFinalizedBatchMeta1, committedBatchMeta1, []*encoding.Chunk{chunk1, chunk2, chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock1)
 
 	block4 := readBlockFromJSON(t, "./testdata/blockTrace_05.json")
 	chunk4 := &encoding.Chunk{Blocks: []*encoding.Block{block4}}
 
-	parentBatchMeta2 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta2 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event1.BatchHash,
 		TotalL1MessagePopped: 11,
 		StateRoot:            event1.StateRoot,
 		WithdrawRoot:         event1.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta2, finalizedBatchMeta1)
+	assert.Equal(t, parentFinalizedBatchMeta2, finalizedBatchMeta1)
 	event2 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(1),
 		BatchHash:    common.HexToHash("0x7f230ce84b4bf86f8ee22ffb5c145e3ef3ddf2a76da4936a33f33cebdb63a48a"),
 		StateRoot:    chunk4.Blocks[len(chunk4.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
-	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentBatchMeta2, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV1),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x01a327088bb2b13151449d8313c281d0006d12e8453e863637b746898b6ad5a6")},
+	}
+	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentFinalizedBatchMeta2, committedBatchMeta2, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock2)
 
-	parentBatchMeta3 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta3 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event2.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event2.StateRoot,
 		WithdrawRoot:         event2.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta3, finalizedBatchMeta2)
+	assert.Equal(t, parentFinalizedBatchMeta3, finalizedBatchMeta2)
 }
 
 func TestValidateBatchCodecv2(t *testing.T) {
-	chainConfig := &params.ChainConfig{BernoulliBlock: big.NewInt(0), CurieBlock: big.NewInt(0)}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{block1}}
 
@@ -666,50 +677,56 @@ func TestValidateBatchCodecv2(t *testing.T) {
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
 	chunk3 := &encoding.Chunk{Blocks: []*encoding.Block{block3}}
 
-	parentBatchMeta1 := &rawdb.FinalizedBatchMeta{}
+	parentFinalizedBatchMeta1 := &rawdb.FinalizedBatchMeta{}
 	event1 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(0),
 		BatchHash:    common.HexToHash("0xaccf37a0b974f2058692d366b2ea85502c99db4a0bcb9b77903b49bf866a463b"),
 		StateRoot:    chunk3.Blocks[len(chunk3.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk3.Blocks[len(chunk3.Blocks)-1].WithdrawRoot,
 	}
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV2),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x018d99636f4b20ccdc1dd11c289eb2a470e2c4dd631b1a7b48a6978805f49d18")},
+	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentBatchMeta1, nil, []*encoding.Chunk{chunk1, chunk2, chunk3}, chainConfig, nil)
+	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentFinalizedBatchMeta1, committedBatchMeta1, []*encoding.Chunk{chunk1, chunk2, chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock1)
 
 	block4 := readBlockFromJSON(t, "./testdata/blockTrace_05.json")
 	chunk4 := &encoding.Chunk{Blocks: []*encoding.Block{block4}}
 
-	parentBatchMeta2 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta2 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event1.BatchHash,
 		TotalL1MessagePopped: 11,
 		StateRoot:            event1.StateRoot,
 		WithdrawRoot:         event1.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta2, finalizedBatchMeta1)
+	assert.Equal(t, parentFinalizedBatchMeta2, finalizedBatchMeta1)
 	event2 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(1),
 		BatchHash:    common.HexToHash("0x62ec61e1fdb334868ffd471df601f6858e692af01d42b5077c805a9fd4558c91"),
 		StateRoot:    chunk4.Blocks[len(chunk4.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
-	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentBatchMeta2, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV2),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x015b4e3d3dcd64cc0eb6a5ad535d7a1844a8c4cdad366ec73557bcc533941370")},
+	}
+	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentFinalizedBatchMeta2, committedBatchMeta2, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock2)
 
-	parentBatchMeta3 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta3 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event2.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event2.StateRoot,
 		WithdrawRoot:         event2.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta3, finalizedBatchMeta2)
+	assert.Equal(t, parentFinalizedBatchMeta3, finalizedBatchMeta2)
 }
 
 func TestValidateBatchCodecv3(t *testing.T) {
-	chainConfig := &params.ChainConfig{BernoulliBlock: big.NewInt(0), CurieBlock: big.NewInt(0), DarwinTime: new(uint64)}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{block1}}
 
@@ -719,7 +736,7 @@ func TestValidateBatchCodecv3(t *testing.T) {
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
 	chunk3 := &encoding.Chunk{Blocks: []*encoding.Block{block3}}
 
-	parentBatchMeta1 := &rawdb.FinalizedBatchMeta{}
+	parentFinalizedBatchMeta1 := &rawdb.FinalizedBatchMeta{}
 	event1 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(0),
 		BatchHash:    common.HexToHash("0x015eb56fb95bf9a06157cfb8389ba7c2b6b08373e22581ac2ba387003708265d"),
@@ -727,46 +744,53 @@ func TestValidateBatchCodecv3(t *testing.T) {
 		WithdrawRoot: chunk3.Blocks[len(chunk3.Blocks)-1].WithdrawRoot,
 	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentBatchMeta1, nil, []*encoding.Chunk{chunk1, chunk2, chunk3}, chainConfig, nil)
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x018d99636f4b20ccdc1dd11c289eb2a470e2c4dd631b1a7b48a6978805f49d18")},
+	}
+
+	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentFinalizedBatchMeta1, committedBatchMeta1, []*encoding.Chunk{chunk1, chunk2, chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock1)
 
 	block4 := readBlockFromJSON(t, "./testdata/blockTrace_05.json")
 	chunk4 := &encoding.Chunk{Blocks: []*encoding.Block{block4}}
 
-	parentBatchMeta2 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta2 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event1.BatchHash,
 		TotalL1MessagePopped: 11,
 		StateRoot:            event1.StateRoot,
 		WithdrawRoot:         event1.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta2, finalizedBatchMeta1)
+	assert.Equal(t, parentFinalizedBatchMeta2, finalizedBatchMeta1)
 	event2 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(1),
 		BatchHash:    common.HexToHash("0x382cb0d507e3d7507f556c52e05f76b05e364ad26205e7f62c95967a19c2f35d"),
 		StateRoot:    chunk4.Blocks[len(chunk4.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
-	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentBatchMeta2, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x015b4e3d3dcd64cc0eb6a5ad535d7a1844a8c4cdad366ec73557bcc533941370")},
+	}
+	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentFinalizedBatchMeta2, committedBatchMeta2, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock2)
 
-	parentBatchMeta3 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta3 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event2.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event2.StateRoot,
 		WithdrawRoot:         event2.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta3, finalizedBatchMeta2)
+	assert.Equal(t, parentFinalizedBatchMeta3, finalizedBatchMeta2)
 }
 
 func TestValidateBatchUpgrades(t *testing.T) {
-	chainConfig := &params.ChainConfig{BernoulliBlock: big.NewInt(3), CurieBlock: big.NewInt(14), DarwinTime: func() *uint64 { t := uint64(1684762320); return &t }()}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	chunk1 := &encoding.Chunk{Blocks: []*encoding.Block{block1}}
 
-	parentBatchMeta1 := &rawdb.FinalizedBatchMeta{}
+	parentFinalizedBatchMeta1 := &rawdb.FinalizedBatchMeta{}
 	event1 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(0),
 		BatchHash:    common.HexToHash("0x4605465b7470c8565b123330d7186805caf9a7f2656d8e9e744b62e14ca22c3d"),
@@ -774,82 +798,97 @@ func TestValidateBatchUpgrades(t *testing.T) {
 		WithdrawRoot: chunk1.Blocks[len(chunk1.Blocks)-1].WithdrawRoot,
 	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentBatchMeta1, nil, []*encoding.Chunk{chunk1}, chainConfig, nil)
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV0),
+		BlobVersionedHashes: nil,
+	}
+
+	endBlock1, finalizedBatchMeta1, err := validateBatch(event1.BatchIndex.Uint64(), event1, parentFinalizedBatchMeta1, committedBatchMeta1, []*encoding.Chunk{chunk1}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(2), endBlock1)
 
 	block2 := readBlockFromJSON(t, "./testdata/blockTrace_03.json")
 	chunk2 := &encoding.Chunk{Blocks: []*encoding.Block{block2}}
 
-	parentBatchMeta2 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta2 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event1.BatchHash,
 		TotalL1MessagePopped: 0,
 		StateRoot:            event1.StateRoot,
 		WithdrawRoot:         event1.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta2, finalizedBatchMeta1)
+	assert.Equal(t, parentFinalizedBatchMeta2, finalizedBatchMeta1)
 	event2 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(1),
 		BatchHash:    common.HexToHash("0xc4af33bce87aa702edc3ad4b7d34730d25719427704e250787f99e0f55049252"),
 		StateRoot:    chunk2.Blocks[len(chunk2.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk2.Blocks[len(chunk2.Blocks)-1].WithdrawRoot,
 	}
-	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentBatchMeta2, nil, []*encoding.Chunk{chunk2}, chainConfig, nil)
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV1),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x01a688c6e137310df38a62f5ad1e5119b8cb0455c386a9a4079b14fe92a239aa")},
+	}
+	endBlock2, finalizedBatchMeta2, err := validateBatch(event2.BatchIndex.Uint64(), event2, parentFinalizedBatchMeta2, committedBatchMeta2, []*encoding.Chunk{chunk2}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(3), endBlock2)
 
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
 	chunk3 := &encoding.Chunk{Blocks: []*encoding.Block{block3}}
 
-	parentBatchMeta3 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta3 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event2.BatchHash,
 		TotalL1MessagePopped: 0,
 		StateRoot:            event2.StateRoot,
 		WithdrawRoot:         event2.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta3, finalizedBatchMeta2)
+	assert.Equal(t, parentFinalizedBatchMeta3, finalizedBatchMeta2)
 	event3 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(2),
 		BatchHash:    common.HexToHash("0x9f87f2de2019ed635f867b1e61be6a607c3174ced096f370fd18556c38833c62"),
 		StateRoot:    chunk3.Blocks[len(chunk3.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk3.Blocks[len(chunk3.Blocks)-1].WithdrawRoot,
 	}
-	endBlock3, finalizedBatchMeta3, err := validateBatch(event3.BatchIndex.Uint64(), event3, parentBatchMeta3, nil, []*encoding.Chunk{chunk3}, chainConfig, nil)
+	committedBatchMeta3 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV1),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x01ea66c4de196d36e2c3a5d7c0045100b9e46ef65be8f7a921ef20e6f2e99ebd")},
+	}
+	endBlock3, finalizedBatchMeta3, err := validateBatch(event3.BatchIndex.Uint64(), event3, parentFinalizedBatchMeta3, committedBatchMeta3, []*encoding.Chunk{chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock3)
 
 	block4 := readBlockFromJSON(t, "./testdata/blockTrace_05.json")
 	chunk4 := &encoding.Chunk{Blocks: []*encoding.Block{block4}}
 
-	parentBatchMeta4 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta4 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event3.BatchHash,
 		TotalL1MessagePopped: 11,
 		StateRoot:            event3.StateRoot,
 		WithdrawRoot:         event3.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta4, finalizedBatchMeta3)
+	assert.Equal(t, parentFinalizedBatchMeta4, finalizedBatchMeta3)
 	event4 := &L1FinalizeBatchEvent{
 		BatchIndex:   big.NewInt(3),
 		BatchHash:    common.HexToHash("0xd33332aef8efbc9a0be4c4694088ac0dd052d2d3ad3ffda5e4c2010825e476bc"),
 		StateRoot:    chunk4.Blocks[len(chunk4.Blocks)-1].Header.Root,
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
-	endBlock4, finalizedBatchMeta4, err := validateBatch(event4.BatchIndex.Uint64(), event4, parentBatchMeta4, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	committedBatchMeta4 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x015b4e3d3dcd64cc0eb6a5ad535d7a1844a8c4cdad366ec73557bcc533941370")},
+	}
+	endBlock4, finalizedBatchMeta4, err := validateBatch(event4.BatchIndex.Uint64(), event4, parentFinalizedBatchMeta4, committedBatchMeta4, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock4)
 
-	parentBatchMeta5 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta5 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event4.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event4.StateRoot,
 		WithdrawRoot:         event4.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta5, finalizedBatchMeta4)
+	assert.Equal(t, parentFinalizedBatchMeta5, finalizedBatchMeta4)
 }
 
 func TestValidateBatchInFinalizeByBundle(t *testing.T) {
-	chainConfig := &params.ChainConfig{BernoulliBlock: big.NewInt(0), CurieBlock: big.NewInt(0), DarwinTime: func() *uint64 { t := uint64(0); return &t }()}
-
 	block1 := readBlockFromJSON(t, "./testdata/blockTrace_02.json")
 	block2 := readBlockFromJSON(t, "./testdata/blockTrace_03.json")
 	block3 := readBlockFromJSON(t, "./testdata/blockTrace_04.json")
@@ -867,29 +906,49 @@ func TestValidateBatchInFinalizeByBundle(t *testing.T) {
 		WithdrawRoot: chunk4.Blocks[len(chunk4.Blocks)-1].WithdrawRoot,
 	}
 
-	endBlock1, finalizedBatchMeta1, err := validateBatch(0, event, &rawdb.FinalizedBatchMeta{}, nil, []*encoding.Chunk{chunk1}, chainConfig, nil)
+	committedBatchMeta1 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x01bbc6b98d7d3783730b6208afac839ad37dcf211b9d9e7c83a5f9d02125ddd7")},
+	}
+
+	committedBatchMeta2 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x01c81e5696e00f1e6e7d76c197f74ed51650147c49c4e6e5b0b702cdcc54352a")},
+	}
+
+	committedBatchMeta3 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x012e15203534ae3f4cbe1b0f58fe6db6e5c29432115a8ece6ef5550bf2ffce4c")},
+	}
+
+	committedBatchMeta4 := &rawdb.CommittedBatchMeta{
+		Version:             uint8(encoding.CodecV3),
+		BlobVersionedHashes: []common.Hash{common.HexToHash("0x015b4e3d3dcd64cc0eb6a5ad535d7a1844a8c4cdad366ec73557bcc533941370")},
+	}
+
+	endBlock1, finalizedBatchMeta1, err := validateBatch(0, event, &rawdb.FinalizedBatchMeta{}, committedBatchMeta1, []*encoding.Chunk{chunk1}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(2), endBlock1)
 
-	endBlock2, finalizedBatchMeta2, err := validateBatch(1, event, finalizedBatchMeta1, nil, []*encoding.Chunk{chunk2}, chainConfig, nil)
+	endBlock2, finalizedBatchMeta2, err := validateBatch(1, event, finalizedBatchMeta1, committedBatchMeta2, []*encoding.Chunk{chunk2}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(3), endBlock2)
 
-	endBlock3, finalizedBatchMeta3, err := validateBatch(2, event, finalizedBatchMeta2, nil, []*encoding.Chunk{chunk3}, chainConfig, nil)
+	endBlock3, finalizedBatchMeta3, err := validateBatch(2, event, finalizedBatchMeta2, committedBatchMeta3, []*encoding.Chunk{chunk3}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(13), endBlock3)
 
-	endBlock4, finalizedBatchMeta4, err := validateBatch(3, event, finalizedBatchMeta3, nil, []*encoding.Chunk{chunk4}, chainConfig, nil)
+	endBlock4, finalizedBatchMeta4, err := validateBatch(3, event, finalizedBatchMeta3, committedBatchMeta4, []*encoding.Chunk{chunk4}, nil)
 	assert.NoError(t, err)
 	assert.Equal(t, uint64(17), endBlock4)
 
-	parentBatchMeta5 := &rawdb.FinalizedBatchMeta{
+	parentFinalizedBatchMeta5 := &rawdb.FinalizedBatchMeta{
 		BatchHash:            event.BatchHash,
 		TotalL1MessagePopped: 42,
 		StateRoot:            event.StateRoot,
 		WithdrawRoot:         event.WithdrawRoot,
 	}
-	assert.Equal(t, parentBatchMeta5, finalizedBatchMeta4)
+	assert.Equal(t, parentFinalizedBatchMeta5, finalizedBatchMeta4)
 }
 
 func readBlockFromJSON(t *testing.T, filename string) *encoding.Block {