internal/era/e2store: refactor e2store to provide ReadAt interface

internal/era/e2store/e2store.go

@@ -17,12 +17,15 @@
 package e2store
 
 import (
+	"encoding/binary"
 	"fmt"
 	"io"
 )
 
-// e2store header size.
+const (
-var headerSize = 8
+	headerSize     = 8
+	valueSizeLimit = 1024 * 1024 * 50
+)
 
 // Entry is a variable-length-data record in an e2store.
 type Entry struct {
@@ -48,21 +51,9 @@ func NewWriter(w io.Writer) *Writer {
 // data (2 bytes). The remaining bytes store b.
 func (w *Writer) Write(typ uint16, b []byte) (int, error) {
 	buf := make([]byte, headerSize+len(b))
-
+	binary.LittleEndian.PutUint16(buf, typ)
-	// type
+	binary.LittleEndian.PutUint32(buf[2:], uint32(len(b)))
-	buf[0] = byte(typ)
-	buf[1] = byte(typ >> 8)
-
-	// length
-	l := len(b)
-	buf[2] = byte(l)
-	buf[3] = byte(l >> 8)
-	buf[4] = byte(l >> 16)
-	buf[5] = byte(l >> 24)
-
-	// value
 	copy(buf[8:], b)
-
 	return w.w.Write(buf)
 }
 
@@ -70,77 +61,126 @@ func (w *Writer) Write(typ uint16, b []byte) (int, error) {
 // For more information on this format, see
 // https://github.com/status-im/nimbus-eth2/blob/stable/docs/e2store.md
 type Reader struct {
-	r io.Reader
+	r      io.ReaderAt
+	offset int64
 }
 
 // NewReader returns a new Reader that reads from r.
-func NewReader(r io.Reader) *Reader {
+func NewReader(r io.ReaderAt) *Reader {
-	return &Reader{r}
+	return &Reader{r, 0}
 }
 
 // Read reads one Entry from r.
-// If the entry is malformed, it returns io.UnexpectedEOF. If there are no
-// entries left to be read, Read returns io.EOF.
 func (r *Reader) Read() (*Entry, error) {
-	b := make([]byte, headerSize)
+	var e Entry
-	if _, err := io.ReadFull(r.r, b); err != nil {
+	n, err := r.ReadAt(&e, r.offset)
+	if err != nil {
 		return nil, err
 	}
+	r.offset += int64(n)
+	return &e, nil
+}
 
-	typ := uint16(b[0])
+// ReadAt reads one Entry from r at the specified offset.
-	typ += uint16(b[1]) << 8
+func (r *Reader) ReadAt(entry *Entry, off int64) (int, error) {
+	typ, length, err := r.ReadMetadataAt(off)
+	if err != nil {
+		return 0, err
+	}
+	entry.Type = typ
 
-	length := uint64(b[2])
+	// Check length bounds.
-	length += uint64(b[3]) << 8
+	if length > valueSizeLimit {
-	length += uint64(b[4]) << 16
+		return headerSize, fmt.Errorf("item larger than item size limit %d: have %d", valueSizeLimit, length)
-	length += uint64(b[5]) << 24
+	}
-
+	if length == 0 {
-	// Check reserved bytes of header.
+		return headerSize, nil
-	if b[6] != 0 || b[7] != 0 {
-		return nil, fmt.Errorf("reserved bytes are non-zero")
 	}
 
+	// Read value.
 	val := make([]byte, length)
-	if _, err := io.ReadFull(r.r, val); err != nil {
+	if n, err := r.r.ReadAt(val, off+headerSize); err != nil {
+		n += headerSize
 		// An entry with a non-zero length should not return EOF when
 		// reading the value.
 		if err == io.EOF {
-			return nil, io.ErrUnexpectedEOF
+			return n, io.ErrUnexpectedEOF
 		}
-		return nil, err
+		return n, err
+	}
+	entry.Value = val
+	return int(headerSize + length), nil
+}
+
+// ReadMetadataAt reads the header metadata at the given offset.
+func (r *Reader) ReadMetadataAt(off int64) (typ uint16, length uint32, err error) {
+	b := make([]byte, headerSize)
+	if n, err := r.r.ReadAt(b, off); err != nil {
+		if err == io.EOF && n > 0 {
+			return 0, 0, io.ErrUnexpectedEOF
+		}
+		return 0, 0, err
+	}
+	typ = binary.LittleEndian.Uint16(b)
+	length = binary.LittleEndian.Uint32(b[2:])
+
+	// Check reserved bytes of header.
+	if b[6] != 0 || b[7] != 0 {
+		return 0, 0, fmt.Errorf("reserved bytes are non-zero")
 	}
 
-	return &Entry{
+	return typ, length, nil
-		Type:  typ,
-		Value: val,
-	}, nil
 }
 
 // Find returns the first entry with the matching type.
-func (r *Reader) Find(typ uint16) (*Entry, error) {
+func (r *Reader) Find(want uint16) (*Entry, error) {
+	var (
+		off    int64
+		typ    uint16
+		length uint32
+		err    error
+	)
 	for {
-		entry, err := r.Read()
+		typ, length, err = r.ReadMetadataAt(off)
 		if err == io.EOF {
 			return nil, io.EOF
 		} else if err != nil {
 			return nil, err
 		}
-		if entry.Type == typ {
+		if typ == want {
-			return entry, nil
+			var e Entry
+			if _, err := r.ReadAt(&e, off); err != nil {
+				return nil, err
 			}
+			return &e, nil
+		}
+		off += int64(headerSize + length)
 	}
 }
 
 // FindAll returns all entries with the matching type.
-func (r *Reader) FindAll(typ uint16) ([]*Entry, error) {
+func (r *Reader) FindAll(want uint16) ([]*Entry, error) {
-	all := make([]*Entry, 0)
+	var (
+		off     int64
+		typ     uint16
+		length  uint32
+		entries []*Entry
+		err     error
+	)
 	for {
-		entry, err := r.Find(typ)
+		typ, length, err = r.ReadMetadataAt(off)
 		if err == io.EOF {
-			return all, io.EOF
+			return entries, nil
 		} else if err != nil {
-			return all, err
+			return entries, err
 		}
-		all = append(all, entry)
+		if typ == want {
+			e := new(Entry)
+			if _, err := r.ReadAt(e, off); err != nil {
+				return entries, err
+			}
+			entries = append(entries, e)
+		}
+		off += int64(headerSize + length)
 	}
 }

internal/era/e2store/e2store_test.go

@@ -26,19 +26,23 @@ import (
 )
 
 func TestEncode(t *testing.T) {
-	for i, tt := range []struct {
+	for _, tt := range []struct {
 		entries []Entry
 		want    string
+		name    string
 	}{
 		{
+			name:    "emptyEntry",
 			entries: []Entry{{0xffff, nil}},
 			want:    "ffff000000000000",
 		},
 		{
+			name:    "beef",
 			entries: []Entry{{42, common.Hex2Bytes("beef")}},
 			want:    "2a00020000000000beef",
 		},
 		{
+			name: "twoEntries",
 			entries: []Entry{
 				{42, common.Hex2Bytes("beef")},
 				{9, common.Hex2Bytes("abcdabcd")},
@@ -46,26 +50,28 @@ func TestEncode(t *testing.T) {
 			want: "2a00020000000000beef0900040000000000abcdabcd",
 		},
 	} {
+		t.Run(tt.name, func(t *testing.T) {
 			var (
 				b = NewWriteSeeker()
 				w = NewWriter(b)
 			)
 			for _, e := range tt.entries {
 				if _, err := w.Write(e.Type, e.Value); err != nil {
-				t.Fatalf("test %d: encoding error: %v", i, err)
+					t.Fatalf("encoding error: %v", err)
 				}
 			}
 			if want, got := common.Hex2Bytes(tt.want), b.Bytes(); !bytes.Equal(want, got) {
-			t.Fatalf("test %d: encoding mismatch (want %s, got %s", i, common.Bytes2Hex(want), common.Bytes2Hex(got))
+				t.Fatalf("encoding mismatch (want %s, got %s", common.Bytes2Hex(want), common.Bytes2Hex(got))
 			}
-		r := NewReader(io.NopCloser(bytes.NewBuffer(b.Bytes())))
+			r := NewReader(bytes.NewReader(b.Bytes()))
 			for _, want := range tt.entries {
 				if got, err := r.Read(); err != nil {
-				t.Fatalf("test %d: decoding error: %v", i, err)
+					t.Fatalf("decoding error: %v", err)
 				} else if got.Type != want.Type || !bytes.Equal(got.Value, want.Value) {
-				t.Fatalf("test %d: decoded entry does not match (want %v, got %v)", i, want, got)
+					t.Fatalf("decoded entry does not match (want %v, got %v)", want, got)
 				}
 			}
+		})
 	}
 }
 
@@ -100,7 +106,7 @@ func TestDecode(t *testing.T) {
 			err:  io.ErrUnexpectedEOF,
 		},
 	} {
-		r := NewReader(io.NopCloser(bytes.NewBuffer(common.Hex2Bytes(tt.have))))
+		r := NewReader(bytes.NewReader(common.Hex2Bytes(tt.have)))
 		if tt.err != nil {
 			if _, err := r.Read(); err != nil && tt.err != nil && err.Error() != tt.err.Error() {
 				t.Fatalf("expected error %v, got %v", tt.err, err)