internal/era: add ReaderAt func so entry value can be read lazily

Co-authored-by: lightclient <lightclient@protonmail.com>
Co-authored-by: Martin Holst Swende <martin@swende.se>

internal/era/e2store/e2store.go

@@ -118,6 +118,24 @@ func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) {
 	return int(headerSize + length), nil
 }
 
+// ReaderAt returns an io.Reader delivering value data for the entry at
+// the specified offset. If the entry type does not match the expected type, an
+// error is returned.
+func (r *Reader) ReaderAt(expectedType uint16, off int64) (io.Reader, int, error) {
+	// problem = need to return length+headerSize not just value length via section reader
+	typ, length, err := r.ReadMetadataAt(off)
+	if err != nil {
+		return nil, headerSize, err
+	}
+	if typ != expectedType {
+		return nil, headerSize, fmt.Errorf("wrong type, want %d have %d", expectedType, typ)
+	}
+	if length > valueSizeLimit {
+		return nil, headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length)
+	}
+	return io.NewSectionReader(r.r, off+headerSize, int64(length)), headerSize + int(length), nil
+}
+
 // LengthAt reads the header at off and returns the total length of the entry,
 // including header.
 func (r *Reader) LengthAt(off int64) (int64, error) {

internal/era/era.go

@@ -17,7 +17,6 @@
 package era
 
 import (
-	"bytes"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -134,7 +133,7 @@ func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
 	if err != nil {
 		return nil, err
 	}
-	r, n, err := newSnappyReader(e.s, off)
+	r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off)
 	if err != nil {
 		return nil, err
 	}
@@ -142,8 +141,8 @@ func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
 	if err := rlp.Decode(r, &header); err != nil {
 		return nil, err
 	}
-	off += int64(n)
+	off += n
-	r, _, err = newSnappyReader(e.s, off)
+	r, _, err = newSnappyReader(e.s, TypeCompressedBody, off)
 	if err != nil {
 		return nil, err
 	}
@@ -170,7 +169,7 @@ func (e *Era) InitialTD() (*big.Int, error) {
 		r      io.Reader
 		header types.Header
 		rawTd  []byte
-		n      int
+		n      int64
 		off    int64
 		err    error
 	)
@@ -179,13 +178,13 @@ func (e *Era) InitialTD() (*big.Int, error) {
 	if off, err = e.readOffset(e.m.start); err != nil {
 		return nil, err
 	}
-	if r, n, err = newSnappyReader(e.s, off); err != nil {
+	if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil {
 		return nil, err
 	}
 	if err := rlp.Decode(r, header); err != nil {
 		return nil, err
 	}
-	off += int64(n)
+	off += n
 
 	// Skip over next two records.
 	for i := 0; i < 2; i++ {
@@ -197,7 +196,7 @@ func (e *Era) InitialTD() (*big.Int, error) {
 	}
 
 	// Read total difficulty after first block.
-	if r, _, err = newReader(e.s, off); err != nil {
+	if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil {
 		return nil, err
 	}
 	if err := rlp.Decode(r, rawTd); err != nil {
@@ -237,23 +236,13 @@ func (e *Era) readOffset(n uint64) (int64, error) {
 	return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
 }
 
-// newReader returns an io.Reader for the e2store entry value at off.
-func newReader(e *e2store.Reader, off int64) (io.Reader, int, error) {
-	var (
-		entry e2store.Entry
-		n     int
-		err   error
-	)
-	if n, err = e.ReadAt(&entry, off); err != nil {
-		return nil, n, err
-	}
-	return bytes.NewReader(entry.Value), n, nil
-}
-
 // newReader returns a snappy.Reader for the e2store entry value at off.
-func newSnappyReader(e *e2store.Reader, off int64) (io.Reader, int, error) {
+func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) {
-	r, n, err := newReader(e, off)
+	r, n, err := e.ReaderAt(expectedType, off)
-	return snappy.NewReader(r), n, err
+	if err != nil {
+		return nil, 0, err
+	}
+	return snappy.NewReader(r), int64(n), err
 }
 
 // clearBuffer zeroes out the buffer.

internal/era/iterator.go

@@ -129,20 +129,20 @@ func (it *RawIterator) Next() bool {
 		it.err = err
 		return false
 	}
-	var n int
+	var n int64
-	if it.Header, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
+	if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil {
 		return true
 	}
-	off += int64(n)
+	off += n
-	if it.Body, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
+	if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil {
 		return true
 	}
-	off += int64(n)
+	off += n
-	if it.Receipts, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
+	if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil {
 		return true
 	}
-	off += int64(n)
+	off += n
-	if it.TotalDifficulty, _, it.err = newReader(it.e.s, off); it.err != nil {
+	if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil {
 		return true
 	}
 	it.next += 1