diff --git a/ethdb/database.go b/ethdb/database.go index 0d752cc8d7..207a07a9a4 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -46,9 +46,14 @@ var ErrTooManyKeys = errors.New("too many keys in deleted range") type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). + // + // A nil start is treated as a key before all keys in the data store; a nil + // end is treated as a key after all keys in the data store. If both is nil + // then the entire data store will be purged. + // // Some implementations of DeleteRange may return ErrTooManyKeys after // partially deleting entries in the given range. - DeleteRange(start, end []byte) error + DeleteRange(start, end []byte) error } // KeyValueStater wraps the Stat method of a backing data store. diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 9ca6a534ea..862ddabb6a 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -401,6 +401,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { db.DeleteRange([]byte(""), []byte("a")) checkRange(1, 999, false) + + addRange(1, 999) + db.DeleteRange(nil, nil) + checkRange(1, 999, false) }) t.Run("BatchDeleteRange", func(t *testing.T) { @@ -437,33 +441,39 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { // Test 1: Basic range deletion in batch addKeys(1, 10) checkKeys(1, 10, true) - + batch := db.NewBatch() - // DeleteRange(start, end) should delete keys where: start <= key < end if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil { t.Fatal(err) } - // Keys shouldn't be deleted until Write is called checkKeys(1, 10, true) - + if err := batch.Write(); err != nil { t.Fatal(err) } - // After Write, keys in range should be deleted // Range is [start, end) - inclusive of start, exclusive of end - checkKeys(1, 2, true) // These should still exist - checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) - checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) - - // Test 2: Mix Put, Delete, and DeleteRange in a batch - // Reset database for next test by adding back deleted keys + checkKeys(1, 2, true) // These should still exist + checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) + checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) + + // Test 2: Delete range with special markers addKeys(3, 7) - - // Verify keys are back + batch = db.NewBatch() + if err := batch.DeleteRange(nil, nil); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + checkKeys(1, 10, false) + + // Test 3: Mix Put, Delete, and DeleteRange in a batch + // Reset database for next test by adding back deleted keys + addKeys(1, 10) checkKeys(1, 10, true) - + // Create a new batch with multiple operations batch = db.NewBatch() if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil { @@ -475,15 +485,13 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil { t.Fatal(err) } - if err := batch.Write(); err != nil { t.Fatal(err) } - // Check results after batch operations // Keys 1-2 should be deleted by DeleteRange checkKeys(1, 2, false) - + // Key 3 should exist (exclusive of end) has, err := db.Has([]byte("3")) if err != nil { @@ -492,7 +500,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !has { t.Fatalf("key 3 should exist after DeleteRange(1,3)") } - + // Key 5 should have a new value val, err := db.Get([]byte("5")) if err != nil { @@ -501,7 +509,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !bytes.Equal(val, []byte("new-val-5")) { t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5") } - + // Key 9 should be deleted has, err = db.Has([]byte("9")) if err != nil { @@ -510,8 +518,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 9 should be deleted") } - - // Test 3: Reset batch + + // Test 4: Reset batch batch.Reset() // Individual deletes work better with both string and numeric comparisons if err := batch.Delete([]byte("8")); err != nil { @@ -526,7 +534,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if err := batch.Write(); err != nil { t.Fatal(err) } - + // Key 8 should be deleted has, err = db.Has([]byte("8")) if err != nil { @@ -535,10 +543,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 8 should be deleted") } - + // Keys 3-7 should still exist checkKeys(3, 7, true) - + // Key 10 should be deleted has, err = db.Has([]byte("10")) if err != nil { @@ -547,8 +555,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 10 should be deleted") } - - // Test 4: Empty range + + // Test 5: Empty range batch = db.NewBatch() if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil { t.Fatal(err) @@ -558,19 +566,19 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } // No existing keys should be affected checkKeys(3, 7, true) - - // Test 5: Test entire keyspace deletion + + // Test 6: Test entire keyspace deletion // First clear any existing keys for i := 1; i <= 100; i++ { db.Delete([]byte(strconv.Itoa(i))) } - + // Then add some fresh test keys addKeys(50, 60) - + // Verify keys exist before deletion checkKeys(50, 60, true) - + batch = db.NewBatch() if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil { t.Fatal(err) @@ -580,37 +588,52 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { } // All keys should be deleted checkKeys(50, 60, false) + + // Test 7: overlapping range deletion + addKeys(50, 60) + batch = db.NewBatch() + if err := batch.DeleteRange([]byte("50"), []byte("55")); err != nil { + t.Fatal(err) + } + if err := batch.DeleteRange([]byte("52"), []byte("58")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + checkKeys(50, 57, false) + checkKeys(58, 60, true) }) - + t.Run("BatchReplayWithDeleteRange", func(t *testing.T) { db := New() defer db.Close() - + // Setup some initial data for i := 1; i <= 10; i++ { if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { t.Fatal(err) } } - + // Create batch with multiple operations including DeleteRange batch1 := db.NewBatch() batch1.Put([]byte("new-key-1"), []byte("new-val-1")) - batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 + batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 batch1.Delete([]byte("8")) batch1.Put([]byte("new-key-2"), []byte("new-val-2")) - + // Create a second batch to replay into batch2 := db.NewBatch() if err := batch1.Replay(batch2); err != nil { t.Fatal(err) } - + // Write the second batch if err := batch2.Write(); err != nil { t.Fatal(err) } - + // Verify results // Original keys 3-6 should be deleted (inclusive of start, exclusive of end) for i := 3; i <= 6; i++ { @@ -622,7 +645,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %d should be deleted", i) } } - + // Key 7 should NOT be deleted (exclusive of end) has, err := db.Has([]byte("7")) if err != nil { @@ -631,7 +654,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if !has { t.Fatalf("key 7 should NOT be deleted (exclusive of end)") } - + // Key 8 should be deleted has, err = db.Has([]byte("8")) if err != nil { @@ -640,7 +663,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { if has { t.Fatalf("key 8 should be deleted") } - + // New keys should be added for _, key := range []string{"new-key-1", "new-key-2"} { has, err := db.Has([]byte(key)) @@ -651,16 +674,16 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %s should exist", key) } } - + // Create a third batch for direct replay to database batch3 := db.NewBatch() - batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 - + batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 + // Replay directly to the database if err := batch3.Replay(db); err != nil { t.Fatal(err) } - + // Verify keys 1-2 are now deleted for i := 1; i <= 2; i++ { has, err := db.Has([]byte(strconv.Itoa(i))) @@ -671,7 +694,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { t.Fatalf("key %d should be deleted after direct replay", i) } } - + // Verify key 3 is NOT deleted (since it's exclusive of end) has, err = db.Has([]byte("3")) if err != nil { @@ -808,16 +831,16 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { for i := 0; i < count; i++ { db.Put([]byte(strconv.Itoa(i)), nil) } - + b.ResetTimer() b.ReportAllocs() - + // Create batch and delete range batch := db.NewBatch() batch.DeleteRange([]byte("0"), []byte("999999999")) batch.Write() } - + b.Run("BatchDeleteRange100", func(b *testing.B) { benchBatchDeleteRange(b, 100) }) @@ -828,42 +851,42 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { benchBatchDeleteRange(b, 10000) }) }) - + b.Run("BatchMixedOps", func(b *testing.B) { benchBatchMixedOps := func(b *testing.B, count int) { db := New() defer db.Close() - + // Prepare initial data for i := 0; i < count; i++ { db.Put([]byte(strconv.Itoa(i)), []byte("val")) } - + b.ResetTimer() b.ReportAllocs() - + // Create batch with mixed operations batch := db.NewBatch() - + // Add some new keys for i := 0; i < count/10; i++ { batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val")) } - + // Delete some individual keys for i := 0; i < count/20; i++ { - batch.Delete([]byte(strconv.Itoa(i*2))) + batch.Delete([]byte(strconv.Itoa(i * 2))) } - + // Delete range of keys rangeStart := count / 2 rangeEnd := count * 3 / 4 batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd))) - + // Write the batch batch.Write() } - + b.Run("BatchMixedOps100", func(b *testing.B) { benchBatchMixedOps(b, 100) }) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index c1523aba1f..736a44d73d 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -220,7 +220,7 @@ func (db *Database) DeleteRange(start, end []byte) error { defer it.Release() var count int - for it.Next() && bytes.Compare(end, it.Key()) > 0 { + for it.Next() && (end == nil || bytes.Compare(end, it.Key()) > 0) { count++ if count > 10000 { // should not block for more than a second if err := batch.Write(); err != nil { @@ -461,48 +461,19 @@ func (b *batch) Delete(key []byte) error { return nil } -// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// DeleteRange removes all keys in the range [start, end) from the batch for +// later committing, inclusive on start, exclusive on end. +// // Note that this is a fallback implementation as leveldb does not natively // support range deletion in batches. It iterates through the database to find // keys in the range and adds them to the batch for deletion. func (b *batch) DeleteRange(start, end []byte) error { - // Special case: empty range - if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) { - return nil - } - - // Special case: delete all keys less than end - if len(start) == 0 { - it := b.db.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if bytes.Compare(key, end) < 0 { - b.b.Delete(key) - b.size += len(key) - } - } - return it.Error() - } - - // Special case: delete all keys greater than or equal to start - if len(end) == 0 { - it := b.db.NewIterator(nil, nil) - defer it.Release() - - for it.Next() { - key := it.Key() - if bytes.Compare(key, start) >= 0 { - b.b.Delete(key) - b.size += len(key) - } - } - return it.Error() - } - // Create an iterator to scan through the keys in the range - it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + slice := &util.Range{ + Start: start, // If nil, it represents the key before all keys + Limit: end, // If nil, it represents the key after all keys + } + it := b.db.NewIterator(slice, nil) defer it.Release() var count int @@ -516,7 +487,6 @@ func (b *batch) DeleteRange(start, end []byte) error { b.b.Delete(key) b.size += len(key) } - if err := it.Error(); err != nil { return err } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index eb49ddbd7b..bd408bfe64 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "sort" - "strconv" "strings" "sync" @@ -124,46 +123,24 @@ func (db *Database) Delete(key []byte) error { } // DeleteRange deletes all of the keys (and values) in the range [start,end) -// (inclusive on start, exclusive on end). +// (inclusive on start, exclusive on end). If the start is nil, it represents +// the key before all keys; if the end is nil, it represents the key after +// all keys. func (db *Database) DeleteRange(start, end []byte) error { db.lock.Lock() defer db.lock.Unlock() + if db.db == nil { return errMemorydbClosed } - - startStr := string(start) - endStr := string(end) - - if startStr == "" && endStr == "" { - return nil // Empty range, do nothing - } - - if startStr == "" { - // Delete all keys less than end - for key := range db.db { - if key < endStr { - delete(db.db, key) - } - } - return nil - } - - if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) { - // Delete all keys greater than or equal to start - for key := range db.db { - if key >= startStr { - delete(db.db, key) - } - } - return nil - } - - // Normal case: delete keys in range [start, end) for key := range db.db { - if key >= startStr && key < endStr { - delete(db.db, key) + if start != nil && key < string(start) { + continue } + if end != nil && key >= string(end) { + continue + } + delete(db.db, key) } return nil } @@ -249,11 +226,12 @@ func (db *Database) Len() int { // keyvalue is a key-value tuple tagged with a deletion field to allow creating // memory-database write batches. type keyvalue struct { - key string - value []byte - delete bool - rangeFrom string - rangeTo string + key string + value []byte + delete bool + + rangeFrom []byte + rangeTo []byte } // batch is a write-only memory batch that commits changes to its host @@ -281,8 +259,8 @@ func (b *batch) Delete(key []byte) error { // DeleteRange removes all keys in the range [start, end) from the batch for later committing. func (b *batch) DeleteRange(start, end []byte) error { b.writes = append(b.writes, keyvalue{ - rangeFrom: string(start), - rangeTo: string(end), + rangeFrom: start, + rangeTo: end, delete: true, }) b.size += len(start) + len(end) @@ -302,46 +280,26 @@ func (b *batch) Write() error { if b.db.db == nil { return errMemorydbClosed } - for _, keyvalue := range b.writes { - if keyvalue.delete { - if keyvalue.key != "" { + for _, entry := range b.writes { + if entry.delete { + if entry.key != "" { // Single key deletion - delete(b.db.db, keyvalue.key) - } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + delete(b.db.db, entry.key) + } else { // Range deletion (inclusive of start, exclusive of end) - // Handle special cases for empty ranges - if keyvalue.rangeFrom == keyvalue.rangeTo { - // Empty range, do nothing - continue - } - - // Check if both are numeric strings for consistent comparison - startNum, startErr := strconv.Atoi(keyvalue.rangeFrom) - endNum, endErr := strconv.Atoi(keyvalue.rangeTo) - - // If both are valid integers, use numeric comparison - if startErr == nil && endErr == nil { - for key := range b.db.db { - if keyNum, err := strconv.Atoi(key); err == nil { - if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) { - delete(b.db.db, key) - } - } + for key := range b.db.db { + if entry.rangeFrom != nil && key < string(entry.rangeFrom) { + continue } - } else { - // Use string comparison - for key := range b.db.db { - // Handle open ranges - if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && - (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) { - delete(b.db.db, key) - } + if entry.rangeTo != nil && key >= string(entry.rangeTo) { + continue } + delete(b.db.db, key) } } continue } - b.db.db[keyvalue.key] = keyvalue.value + b.db.db[entry.key] = entry.value } return nil } @@ -354,17 +312,17 @@ func (b *batch) Reset() { // Replay replays the batch contents. func (b *batch) Replay(w ethdb.KeyValueWriter) error { - for _, keyvalue := range b.writes { - if keyvalue.delete { - if keyvalue.key != "" { + for _, entry := range b.writes { + if entry.delete { + if entry.key != "" { // Single key deletion - if err := w.Delete([]byte(keyvalue.key)); err != nil { + if err := w.Delete([]byte(entry.key)); err != nil { return err } - } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + } else { // Range deletion if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok { - if err := rangeDeleter.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { + if err := rangeDeleter.DeleteRange(entry.rangeFrom, entry.rangeTo); err != nil { return err } } else { @@ -373,7 +331,7 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } continue } - if err := w.Put([]byte(keyvalue.key), keyvalue.value); err != nil { + if err := w.Put([]byte(entry.key), entry.value); err != nil { return err } } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index c92718d0d6..09062d26fc 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -403,9 +403,16 @@ func (d *Database) Delete(key []byte) error { func (d *Database) DeleteRange(start, end []byte) error { d.quitLock.RLock() defer d.quitLock.RUnlock() + if d.closed { return pebble.ErrClosed } + // There is no special flag to represent the end of key range + // in pebble(nil in leveldb). Use an ugly hack to construct a + // large key to represent it. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } return d.db.DeleteRange(start, end, d.writeOptions) } @@ -619,8 +626,15 @@ func (b *batch) Delete(key []byte) error { return nil } -// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// DeleteRange removes all keys in the range [start, end) from the batch for +// later committing, inclusive on start, exclusive on end. func (b *batch) DeleteRange(start, end []byte) error { + // There is no special flag to represent the end of key range + // in pebble(nil in leveldb). Use an ugly hack to construct a + // large key to represent it. + if end == nil { + end = bytes.Repeat([]byte{0xff}, 32) + } if err := b.b.DeleteRange(start, end, nil); err != nil { return err }