ethdb, rawdb, leveldb, memorydb: implement iterators with prefix and start

core/rawdb/table.go

@@ -113,17 +113,21 @@ func (t *table) NewIterator() ethdb.Iterator {
 // database content starting at a particular initial key (or after, if it does
 // not exist).
 func (t *table) NewIteratorWithStart(start []byte) ethdb.Iterator {
-	iter := t.db.NewIteratorWithStart(append([]byte(t.prefix), start...))
+	return t.NewIteratorWith(nil, start)
-	return &tableIterator{
-		iter:   iter,
-		prefix: t.prefix,
-	}
 }
 
 // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
 // of database content with a particular key prefix.
 func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
-	iter := t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...))
+	return t.NewIteratorWith(prefix, nil)
+}
+
+// NewIteratorWith creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix, starting at a particular
+// initial key (or after, if it does not exist).
+func (t *table) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator {
+	innerPrefix := append([]byte(t.prefix), prefix...)
+	iter := t.db.NewIteratorWith(innerPrefix, start)
 	return &tableIterator{
 		iter:   iter,
 		prefix: t.prefix,

core/rawdb/table_test.go

@@ -127,7 +127,8 @@ func testTableDatabase(t *testing.T, prefix string) {
 	iter.Release()
 
 	// Test iterators with start point
-	iter = db.NewIteratorWithStart([]byte{0xff, 0xff, 0x02})
+	//iter = db.NewIteratorWithStart([]byte{0xff, 0xff, 0x02})
+	iter = db.NewIteratorWith(nil,[]byte{0xff, 0xff, 0x02})
 	index = 4
 	for iter.Next() {
 		key, value := iter.Key(), iter.Value()

core/state/snapshot/disklayer_test.go

@@ -18,11 +18,14 @@ package snapshot
 
 import (
 	"bytes"
+	"github.com/ethereum/go-ethereum/ethdb/leveldb"
+	"io/ioutil"
 	"testing"
 
 	"github.com/VictoriaMetrics/fastcache"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 )
 
@@ -435,15 +438,35 @@ func TestDiskMidAccountPartialMerge(t *testing.T) {
 	// TODO(@karalabe) ?
 }
 
+func tempDB() (ethdb.Database, error) {
+	dir, err := ioutil.TempDir("", "disklayer-test")
+	if err != nil {
+		return nil, err
+	}
+	diskdb, err := leveldb.New(dir, 256, 0, "")
+	if err != nil {
+		return nil, err
+	}
+	return rawdb.NewDatabase(diskdb), nil
+}
+
 // TestDiskSeek tests that seek-operations work on the disk layer
 func TestDiskSeek(t *testing.T) {
 	// Create some accounts in the disk layer
-	db := memorydb.New()
+	//db := memorydb.New()
+	db ,err := tempDB()
+	if err != nil {
+		t.Fatal( err)
+	}
 	// Fill even keys [0,2,4...]
 	for i := 0; i < 0xff; i += 2 {
 		acc := common.Hash{byte(i)}
 		rawdb.WriteAccountSnapshot(db, acc, acc[:])
 	}
+	// Add an 'higher' key, with incorrect (higher) prefix
+	highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1}
+	db.Put(highKey, []byte{0xff, 0xff})
+
 	baseRoot := randomHash()
 	rawdb.WriteSnapshotRoot(db, baseRoot)
 
@@ -475,16 +498,17 @@ func TestDiskSeek(t *testing.T) {
 		}
 		count := 0
 		for it.Next() {
-			count++
 			k, v, err := it.Hash()[0], it.Account()[0], it.Error()
 			if err != nil {
-				t.Fatalf("test %d, error: %v", i, err)
+				t.Fatalf("test %d, item %d, error: %v", i, count, err)
 			}
-			if k != tc.expkey {
+			// First item in iterator should have the expected key
-				t.Fatalf("test %d, got %v exp %v", i, k, tc.expkey)
+			if count == 0 && k != tc.expkey {
+				t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey)
 			}
+			count++
 			if v != k {
-				t.Fatalf("test %d, value wrong, got %v exp %v", i, v, k)
+				t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k)
 			}
 		}
 	}

core/state/snapshot/iterator.go

@@ -148,10 +148,10 @@ type diskAccountIterator struct {
 
 // AccountIterator creates an account iterator over a disk layer.
 func (dl *diskLayer) AccountIterator(seek common.Hash) AccountIterator {
-	pos := append(rawdb.SnapshotAccountPrefix, common.TrimRightZeroes(seek[:])...)
+	pos := common.TrimRightZeroes(seek[:])
 	return &diskAccountIterator{
 		layer: dl,
-		it:    dl.diskdb.NewIteratorWithPrefix(pos),
+		it:    dl.diskdb.NewIteratorWith(rawdb.SnapshotAccountPrefix, pos),
 	}
 }
 

core/state/snapshot/wipe.go

@@ -92,7 +92,7 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 	// Iterate over the key-range and delete all of them
 	start, logged := time.Now(), time.Now()
 
-	it := db.NewIteratorWithStart(prefix)
+	it := db.NewIteratorWith(prefix, nil)
 	for it.Next() {
 		// Skip any keys with the correct prefix but wrong lenth (trie nodes)
 		key := it.Key()
@@ -113,7 +113,8 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int
 				return err
 			}
 			batch.Reset()
-			it = db.NewIteratorWithStart(key)
+			seekPos := bytes.Trim(key, prefix)
+			it = db.NewIteratorWith(prefix, seekPos)
 
 			if time.Since(logged) > 8*time.Second {
 				log.Info("Deleting state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start)))

ethdb/dbtest/testsuite.go

@@ -32,31 +32,32 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		tests := []struct {
 			content map[string]string
 			prefix  string
+			start string
 			order   []string
 		}{
 			// Empty databases should be iterable
-			{map[string]string{}, "", nil},
+			{map[string]string{}, "","", nil},
-			{map[string]string{}, "non-existent-prefix", nil},
+			{map[string]string{}, "non-existent-prefix","", nil},
 
 			// Single-item databases should be iterable
-			{map[string]string{"key": "val"}, "", []string{"key"}},
+			{map[string]string{"key": "val"}, "","", []string{"key"}},
-			{map[string]string{"key": "val"}, "k", []string{"key"}},
+			{map[string]string{"key": "val"}, "k","", []string{"key"}},
-			{map[string]string{"key": "val"}, "l", nil},
+			{map[string]string{"key": "val"}, "l","", nil},
 
 			// Multi-item databases should be fully iterable
 			{
 				map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"},
-				"",
+				"","",
 				[]string{"k1", "k2", "k3", "k4", "k5"},
 			},
 			{
 				map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"},
-				"k",
+				"k","",
 				[]string{"k1", "k2", "k3", "k4", "k5"},
 			},
 			{
 				map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"},
-				"l",
+				"l","",
 				nil,
 			},
 			// Multi-item databases should be prefix-iterable
@@ -65,7 +66,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 					"ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3",
 					"kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3",
 				},
-				"ka",
+				"ka","",
 				[]string{"ka1", "ka2", "ka3", "ka4", "ka5"},
 			},
 			{
@@ -73,7 +74,24 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 					"ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3",
 					"kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3",
 				},
-				"kc",
+				"kc","",
+				nil,
+			},
+			// Multi-item databases should be prefix-iterable with start position
+			{
+				map[string]string{
+					"ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3",
+					"kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3",
+				},
+				"ka","3",
+				[]string{"ka3", "ka4", "ka5"},
+			},
+			{
+				map[string]string{
+					"ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3",
+					"kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3",
+				},
+				"ka","8",
 				nil,
 			},
 		}
@@ -86,7 +104,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 				}
 			}
 			// Iterate over the database with the given configs and verify the results
-			it, idx := db.NewIteratorWithPrefix([]byte(tt.prefix)), 0
+			it, idx := db.NewIteratorWith([]byte(tt.prefix), []byte(tt.start)), 0
 			for it.Next() {
 				if len(tt.order) <= idx {
 					t.Errorf("test %d: prefix=%q more items than expected: checking idx=%d (key %q), expecting len=%d", i, tt.prefix, idx, it.Key(), len(tt.order))

ethdb/iterator.go

@@ -63,4 +63,9 @@ type Iteratee interface {
 	// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
 	// of database content with a particular key prefix.
 	NewIteratorWithPrefix(prefix []byte) Iterator
+
+	// NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
+	// of database content with a particular key prefix, starting at a particular
+	//initial key (or after, if it does not exist).
+	NewIteratorWith(prefix []byte, start []byte) Iterator
 }

ethdb/leveldb/leveldb.go

@@ -202,6 +202,13 @@ func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
 	return db.db.NewIterator(util.BytesPrefix(prefix), nil)
 }
 
+// NewIteratorWith creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix, starting at a particular
+// initial key (or after, if it does not exist).
+func (db *Database) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator {
+	return db.db.NewIterator(BytesPrefixRange(prefix, start), nil)
+}
+
 // Stat returns a particular internal stat of the database.
 func (db *Database) Stat(property string) (string, error) {
 	return db.db.GetProperty(property)
@@ -488,3 +495,12 @@ func (r *replayer) Delete(key []byte) {
 	}
 	r.failure = r.writer.Delete(key)
 }
+
+// BytesPrefixRange returns key range that satisfy
+// - the given prefix, and
+// - the given seek position
+func BytesPrefixRange(prefix, start []byte) *util.Range {
+	r := util.BytesPrefix(prefix)
+	r.Start = append(r.Start, start...)
+	return r
+}

ethdb/memorydb/memorydb.go

@@ -139,45 +139,37 @@ func (db *Database) NewIterator() ethdb.Iterator {
 // database content starting at a particular initial key (or after, if it does
 // not exist).
 func (db *Database) NewIteratorWithStart(start []byte) ethdb.Iterator {
-	db.lock.RLock()
+	return db.NewIteratorWith(nil, start)
-	defer db.lock.RUnlock()
-
-	var (
-		st     = string(start)
-		keys   = make([]string, 0, len(db.db))
-		values = make([][]byte, 0, len(db.db))
-	)
-	// Collect the keys from the memory database corresponding to the given start
-	for key := range db.db {
-		if key >= st {
-			keys = append(keys, key)
-		}
-	}
-	// Sort the items and retrieve the associated values
-	sort.Strings(keys)
-	for _, key := range keys {
-		values = append(values, db.db[key])
-	}
-	return &iterator{
-		keys:   keys,
-		values: values,
-	}
 }
 
 // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset
 // of database content with a particular key prefix.
 func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator {
+	return db.NewIteratorWith(prefix, nil)
+}
+
+// NewIteratorWith creates a binary-alphabetical iterator over a subset
+// of database content with a particular key prefix, starting at a particular
+// initial key (or after, if it does not exist).
+// Note: This method assumes that the prefix is NOT part of the start, so there's
+// no need for the caller to prepend the prefix to the start
+func (db *Database) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator {
 	db.lock.RLock()
 	defer db.lock.RUnlock()
 
 	var (
 		pr     = string(prefix)
+		st     = string(append(prefix, start...))
 		keys   = make([]string, 0, len(db.db))
 		values = make([][]byte, 0, len(db.db))
 	)
 	// Collect the keys from the memory database corresponding to the given prefix
+	// and start
 	for key := range db.db {
-		if strings.HasPrefix(key, pr) {
+		if !strings.HasPrefix(key, pr) {
+			continue
+		}
+		if key >= st {
 			keys = append(keys, key)
 		}
 	}