From 977908765541abe73e0692a0704ef8b6e31e0e41 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Mar 2020 11:58:05 +0100 Subject: [PATCH] ethdb, rawdb, leveldb, memorydb: implement iterators with prefix and start --- core/rawdb/table.go | 16 +++++++---- core/rawdb/table_test.go | 3 +- core/state/snapshot/disklayer_test.go | 36 ++++++++++++++++++++---- core/state/snapshot/iterator.go | 4 +-- core/state/snapshot/wipe.go | 5 ++-- ethdb/dbtest/testsuite.go | 40 +++++++++++++++++++-------- ethdb/iterator.go | 5 ++++ ethdb/leveldb/leveldb.go | 16 +++++++++++ ethdb/memorydb/memorydb.go | 40 +++++++++++---------------- 9 files changed, 113 insertions(+), 52 deletions(-) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 092341fbfe..a63536c416 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -113,17 +113,21 @@ func (t *table) NewIterator() ethdb.Iterator { // database content starting at a particular initial key (or after, if it does // not exist). func (t *table) NewIteratorWithStart(start []byte) ethdb.Iterator { - iter := t.db.NewIteratorWithStart(append([]byte(t.prefix), start...)) - return &tableIterator{ - iter: iter, - prefix: t.prefix, - } + return t.NewIteratorWith(nil, start) } // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix. func (t *table) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { - iter := t.db.NewIteratorWithPrefix(append([]byte(t.prefix), prefix...)) + return t.NewIteratorWith(prefix, nil) +} + +// NewIteratorWith creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix, starting at a particular +// initial key (or after, if it does not exist). +func (t *table) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator { + innerPrefix := append([]byte(t.prefix), prefix...) + iter := t.db.NewIteratorWith(innerPrefix, start) return &tableIterator{ iter: iter, prefix: t.prefix, diff --git a/core/rawdb/table_test.go b/core/rawdb/table_test.go index 977eb4bf05..dd7936cef0 100644 --- a/core/rawdb/table_test.go +++ b/core/rawdb/table_test.go @@ -127,7 +127,8 @@ func testTableDatabase(t *testing.T, prefix string) { iter.Release() // Test iterators with start point - iter = db.NewIteratorWithStart([]byte{0xff, 0xff, 0x02}) + //iter = db.NewIteratorWithStart([]byte{0xff, 0xff, 0x02}) + iter = db.NewIteratorWith(nil,[]byte{0xff, 0xff, 0x02}) index = 4 for iter.Next() { key, value := iter.Key(), iter.Value() diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 26fe8a5339..94760d9f9b 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -18,11 +18,14 @@ package snapshot import ( "bytes" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "io/ioutil" "testing" "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" ) @@ -435,15 +438,35 @@ func TestDiskMidAccountPartialMerge(t *testing.T) { // TODO(@karalabe) ? } +func tempDB() (ethdb.Database, error) { + dir, err := ioutil.TempDir("", "disklayer-test") + if err != nil { + return nil, err + } + diskdb, err := leveldb.New(dir, 256, 0, "") + if err != nil { + return nil, err + } + return rawdb.NewDatabase(diskdb), nil +} + // TestDiskSeek tests that seek-operations work on the disk layer func TestDiskSeek(t *testing.T) { // Create some accounts in the disk layer - db := memorydb.New() + //db := memorydb.New() + db ,err := tempDB() + if err != nil { + t.Fatal( err) + } // Fill even keys [0,2,4...] for i := 0; i < 0xff; i += 2 { acc := common.Hash{byte(i)} rawdb.WriteAccountSnapshot(db, acc, acc[:]) } + // Add an 'higher' key, with incorrect (higher) prefix + highKey := []byte{rawdb.SnapshotAccountPrefix[0] + 1} + db.Put(highKey, []byte{0xff, 0xff}) + baseRoot := randomHash() rawdb.WriteSnapshotRoot(db, baseRoot) @@ -475,16 +498,17 @@ func TestDiskSeek(t *testing.T) { } count := 0 for it.Next() { - count++ k, v, err := it.Hash()[0], it.Account()[0], it.Error() if err != nil { - t.Fatalf("test %d, error: %v", i, err) + t.Fatalf("test %d, item %d, error: %v", i, count, err) } - if k != tc.expkey { - t.Fatalf("test %d, got %v exp %v", i, k, tc.expkey) + // First item in iterator should have the expected key + if count == 0 && k != tc.expkey { + t.Fatalf("test %d, item %d, got %v exp %v", i, count, k, tc.expkey) } + count++ if v != k { - t.Fatalf("test %d, value wrong, got %v exp %v", i, v, k) + t.Fatalf("test %d, item %d, value wrong, got %v exp %v", i, count, v, k) } } } diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index e6d08c8106..80391796b8 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -148,10 +148,10 @@ type diskAccountIterator struct { // AccountIterator creates an account iterator over a disk layer. func (dl *diskLayer) AccountIterator(seek common.Hash) AccountIterator { - pos := append(rawdb.SnapshotAccountPrefix, common.TrimRightZeroes(seek[:])...) + pos := common.TrimRightZeroes(seek[:]) return &diskAccountIterator{ layer: dl, - it: dl.diskdb.NewIteratorWithPrefix(pos), + it: dl.diskdb.NewIteratorWith(rawdb.SnapshotAccountPrefix, pos), } } diff --git a/core/state/snapshot/wipe.go b/core/state/snapshot/wipe.go index 052af6f1f1..52d32e4268 100644 --- a/core/state/snapshot/wipe.go +++ b/core/state/snapshot/wipe.go @@ -92,7 +92,7 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int // Iterate over the key-range and delete all of them start, logged := time.Now(), time.Now() - it := db.NewIteratorWithStart(prefix) + it := db.NewIteratorWith(prefix, nil) for it.Next() { // Skip any keys with the correct prefix but wrong lenth (trie nodes) key := it.Key() @@ -113,7 +113,8 @@ func wipeKeyRange(db ethdb.KeyValueStore, kind string, prefix []byte, keylen int return err } batch.Reset() - it = db.NewIteratorWithStart(key) + seekPos := bytes.Trim(key, prefix) + it = db.NewIteratorWith(prefix, seekPos) if time.Since(logged) > 8*time.Second { log.Info("Deleting state snapshot leftovers", "kind", kind, "wiped", items, "elapsed", common.PrettyDuration(time.Since(start))) diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index dce2ba2a1f..b6dcf92707 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -32,31 +32,32 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { tests := []struct { content map[string]string prefix string + start string order []string }{ // Empty databases should be iterable - {map[string]string{}, "", nil}, - {map[string]string{}, "non-existent-prefix", nil}, + {map[string]string{}, "","", nil}, + {map[string]string{}, "non-existent-prefix","", nil}, // Single-item databases should be iterable - {map[string]string{"key": "val"}, "", []string{"key"}}, - {map[string]string{"key": "val"}, "k", []string{"key"}}, - {map[string]string{"key": "val"}, "l", nil}, + {map[string]string{"key": "val"}, "","", []string{"key"}}, + {map[string]string{"key": "val"}, "k","", []string{"key"}}, + {map[string]string{"key": "val"}, "l","", nil}, // Multi-item databases should be fully iterable { map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "", + "","", []string{"k1", "k2", "k3", "k4", "k5"}, }, { map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "k", + "k","", []string{"k1", "k2", "k3", "k4", "k5"}, }, { map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "l", + "l","", nil, }, // Multi-item databases should be prefix-iterable @@ -65,7 +66,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", }, - "ka", + "ka","", []string{"ka1", "ka2", "ka3", "ka4", "ka5"}, }, { @@ -73,7 +74,24 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", }, - "kc", + "kc","", + nil, + }, + // Multi-item databases should be prefix-iterable with start position + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "ka","3", + []string{"ka3", "ka4", "ka5"}, + }, + { + map[string]string{ + "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", + "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", + }, + "ka","8", nil, }, } @@ -86,7 +104,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } } // Iterate over the database with the given configs and verify the results - it, idx := db.NewIteratorWithPrefix([]byte(tt.prefix)), 0 + it, idx := db.NewIteratorWith([]byte(tt.prefix), []byte(tt.start)), 0 for it.Next() { if len(tt.order) <= idx { t.Errorf("test %d: prefix=%q more items than expected: checking idx=%d (key %q), expecting len=%d", i, tt.prefix, idx, it.Key(), len(tt.order)) diff --git a/ethdb/iterator.go b/ethdb/iterator.go index 419e9bdfc2..3d95f867d1 100644 --- a/ethdb/iterator.go +++ b/ethdb/iterator.go @@ -63,4 +63,9 @@ type Iteratee interface { // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix. NewIteratorWithPrefix(prefix []byte) Iterator + + // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset + // of database content with a particular key prefix, starting at a particular + //initial key (or after, if it does not exist). + NewIteratorWith(prefix []byte, start []byte) Iterator } diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 378d4c3cd2..cd7458f2c7 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -202,6 +202,13 @@ func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { return db.db.NewIterator(util.BytesPrefix(prefix), nil) } +// NewIteratorWith creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix, starting at a particular +// initial key (or after, if it does not exist). +func (db *Database) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator { + return db.db.NewIterator(BytesPrefixRange(prefix, start), nil) +} + // Stat returns a particular internal stat of the database. func (db *Database) Stat(property string) (string, error) { return db.db.GetProperty(property) @@ -488,3 +495,12 @@ func (r *replayer) Delete(key []byte) { } r.failure = r.writer.Delete(key) } + +// BytesPrefixRange returns key range that satisfy +// - the given prefix, and +// - the given seek position +func BytesPrefixRange(prefix, start []byte) *util.Range { + r := util.BytesPrefix(prefix) + r.Start = append(r.Start, start...) + return r +} diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 346edc4381..2d254c8551 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -139,45 +139,37 @@ func (db *Database) NewIterator() ethdb.Iterator { // database content starting at a particular initial key (or after, if it does // not exist). func (db *Database) NewIteratorWithStart(start []byte) ethdb.Iterator { - db.lock.RLock() - defer db.lock.RUnlock() - - var ( - st = string(start) - keys = make([]string, 0, len(db.db)) - values = make([][]byte, 0, len(db.db)) - ) - // Collect the keys from the memory database corresponding to the given start - for key := range db.db { - if key >= st { - keys = append(keys, key) - } - } - // Sort the items and retrieve the associated values - sort.Strings(keys) - for _, key := range keys { - values = append(values, db.db[key]) - } - return &iterator{ - keys: keys, - values: values, - } + return db.NewIteratorWith(nil, start) } // NewIteratorWithPrefix creates a binary-alphabetical iterator over a subset // of database content with a particular key prefix. func (db *Database) NewIteratorWithPrefix(prefix []byte) ethdb.Iterator { + return db.NewIteratorWith(prefix, nil) +} + +// NewIteratorWith creates a binary-alphabetical iterator over a subset +// of database content with a particular key prefix, starting at a particular +// initial key (or after, if it does not exist). +// Note: This method assumes that the prefix is NOT part of the start, so there's +// no need for the caller to prepend the prefix to the start +func (db *Database) NewIteratorWith(prefix []byte, start []byte) ethdb.Iterator { db.lock.RLock() defer db.lock.RUnlock() var ( pr = string(prefix) + st = string(append(prefix, start...)) keys = make([]string, 0, len(db.db)) values = make([][]byte, 0, len(db.db)) ) // Collect the keys from the memory database corresponding to the given prefix + // and start for key := range db.db { - if strings.HasPrefix(key, pr) { + if !strings.HasPrefix(key, pr) { + continue + } + if key >= st { keys = append(keys, key) } }