diff --git a/ethdb/batch.go b/ethdb/batch.go index 541f40c838..f3c2ca555c 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -25,6 +25,10 @@ const IdealBatchSize = 100 * 1024 type Batch interface { KeyValueWriter + // DeleteRange deletes all of the keys (and values) in the range [start,end) + // (inclusive on start, exclusive on end). + DeleteRange(start, end []byte) error + // ValueSize retrieves the amount of data queued up for writing. ValueSize() int @@ -53,8 +57,9 @@ type Batcher interface { type HookedBatch struct { Batch - OnPut func(key []byte, value []byte) // Callback if a key is inserted - OnDelete func(key []byte) // Callback if a key is deleted + OnPut func(key []byte, value []byte) // Callback if a key is inserted + OnDelete func(key []byte) // Callback if a key is deleted + OnDeleteRange func(start, end []byte) // Callback if a range of keys is deleted } // Put inserts the given value into the key-value data store. @@ -72,3 +77,11 @@ func (b HookedBatch) Delete(key []byte) error { } return b.Batch.Delete(key) } + +// DeleteRange removes all keys in the range [start, end) from the key-value data store. +func (b HookedBatch) DeleteRange(start, end []byte) error { + if b.OnDeleteRange != nil { + b.OnDeleteRange(start, end) + } + return b.Batch.DeleteRange(start, end) +} diff --git a/ethdb/database.go b/ethdb/database.go index 7f421752c4..2b9cba4e69 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -38,19 +38,16 @@ type KeyValueWriter interface { // Delete removes the key from the key-value data store. Delete(key []byte) error -} -var ErrTooManyKeys = errors.New("too many keys in deleted range") - -// KeyValueRangeDeleter wraps the DeleteRange method of a backing data store. -type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). // Some implementations of DeleteRange may return ErrTooManyKeys after // partially deleting entries in the given range. - DeleteRange(start, end []byte) error + DeleteRange(start, end []byte) error } +var ErrTooManyKeys = errors.New("too many keys in deleted range") + // KeyValueStater wraps the Stat method of a backing data store. type KeyValueStater interface { // Stat returns the statistic data of the database. @@ -83,7 +80,6 @@ type KeyValueStore interface { KeyValueWriter KeyValueStater KeyValueSyncer - KeyValueRangeDeleter Batcher Iteratee Compacter diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index 52e6b287cf..9ca6a534ea 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -402,6 +402,285 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { db.DeleteRange([]byte(""), []byte("a")) checkRange(1, 999, false) }) + + t.Run("BatchDeleteRange", func(t *testing.T) { + db := New() + defer db.Close() + + // Helper to add keys + addKeys := func(start, stop int) { + for i := start; i <= stop; i++ { + if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + } + + // Helper to check if keys exist + checkKeys := func(start, stop int, shouldExist bool) { + for i := start; i <= stop; i++ { + key := []byte(strconv.Itoa(i)) + has, err := db.Has(key) + if err != nil { + t.Fatal(err) + } + if has != shouldExist { + if shouldExist { + t.Fatalf("key %s should exist but doesn't", key) + } else { + t.Fatalf("key %s shouldn't exist but does", key) + } + } + } + } + + // Test 1: Basic range deletion in batch + addKeys(1, 10) + checkKeys(1, 10, true) + + batch := db.NewBatch() + // DeleteRange(start, end) should delete keys where: start <= key < end + if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil { + t.Fatal(err) + } + + // Keys shouldn't be deleted until Write is called + checkKeys(1, 10, true) + + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // After Write, keys in range should be deleted + // Range is [start, end) - inclusive of start, exclusive of end + checkKeys(1, 2, true) // These should still exist + checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive) + checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive) + + // Test 2: Mix Put, Delete, and DeleteRange in a batch + // Reset database for next test by adding back deleted keys + addKeys(3, 7) + + // Verify keys are back + checkKeys(1, 10, true) + + // Create a new batch with multiple operations + batch = db.NewBatch() + if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("9")); err != nil { + t.Fatal(err) + } + if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil { + t.Fatal(err) + } + + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // Check results after batch operations + // Keys 1-2 should be deleted by DeleteRange + checkKeys(1, 2, false) + + // Key 3 should exist (exclusive of end) + has, err := db.Has([]byte("3")) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key 3 should exist after DeleteRange(1,3)") + } + + // Key 5 should have a new value + val, err := db.Get([]byte("5")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(val, []byte("new-val-5")) { + t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5") + } + + // Key 9 should be deleted + has, err = db.Has([]byte("9")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 9 should be deleted") + } + + // Test 3: Reset batch + batch.Reset() + // Individual deletes work better with both string and numeric comparisons + if err := batch.Delete([]byte("8")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("10")); err != nil { + t.Fatal(err) + } + if err := batch.Delete([]byte("11")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + + // Key 8 should be deleted + has, err = db.Has([]byte("8")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 8 should be deleted") + } + + // Keys 3-7 should still exist + checkKeys(3, 7, true) + + // Key 10 should be deleted + has, err = db.Has([]byte("10")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 10 should be deleted") + } + + // Test 4: Empty range + batch = db.NewBatch() + if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + // No existing keys should be affected + checkKeys(3, 7, true) + + // Test 5: Test entire keyspace deletion + // First clear any existing keys + for i := 1; i <= 100; i++ { + db.Delete([]byte(strconv.Itoa(i))) + } + + // Then add some fresh test keys + addKeys(50, 60) + + // Verify keys exist before deletion + checkKeys(50, 60, true) + + batch = db.NewBatch() + if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil { + t.Fatal(err) + } + if err := batch.Write(); err != nil { + t.Fatal(err) + } + // All keys should be deleted + checkKeys(50, 60, false) + }) + + t.Run("BatchReplayWithDeleteRange", func(t *testing.T) { + db := New() + defer db.Close() + + // Setup some initial data + for i := 1; i <= 10; i++ { + if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil { + t.Fatal(err) + } + } + + // Create batch with multiple operations including DeleteRange + batch1 := db.NewBatch() + batch1.Put([]byte("new-key-1"), []byte("new-val-1")) + batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7 + batch1.Delete([]byte("8")) + batch1.Put([]byte("new-key-2"), []byte("new-val-2")) + + // Create a second batch to replay into + batch2 := db.NewBatch() + if err := batch1.Replay(batch2); err != nil { + t.Fatal(err) + } + + // Write the second batch + if err := batch2.Write(); err != nil { + t.Fatal(err) + } + + // Verify results + // Original keys 3-6 should be deleted (inclusive of start, exclusive of end) + for i := 3; i <= 6; i++ { + has, err := db.Has([]byte(strconv.Itoa(i))) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key %d should be deleted", i) + } + } + + // Key 7 should NOT be deleted (exclusive of end) + has, err := db.Has([]byte("7")) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key 7 should NOT be deleted (exclusive of end)") + } + + // Key 8 should be deleted + has, err = db.Has([]byte("8")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 8 should be deleted") + } + + // New keys should be added + for _, key := range []string{"new-key-1", "new-key-2"} { + has, err := db.Has([]byte(key)) + if err != nil { + t.Fatal(err) + } + if !has { + t.Fatalf("key %s should exist", key) + } + } + + // Create a third batch for direct replay to database + batch3 := db.NewBatch() + batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3 + + // Replay directly to the database + if err := batch3.Replay(db); err != nil { + t.Fatal(err) + } + + // Verify keys 1-2 are now deleted + for i := 1; i <= 2; i++ { + has, err := db.Has([]byte(strconv.Itoa(i))) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key %d should be deleted after direct replay", i) + } + } + + // Verify key 3 is NOT deleted (since it's exclusive of end) + has, err = db.Has([]byte("3")) + if err != nil { + t.Fatal(err) + } + if has { + t.Fatalf("key 3 should still be deleted from previous operation") + } + }) } // BenchDatabaseSuite runs a suite of benchmarks against a KeyValueStore database @@ -520,6 +799,81 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { benchDeleteRange(b, 10000) }) }) + b.Run("BatchDeleteRange", func(b *testing.B) { + benchBatchDeleteRange := func(b *testing.B, count int) { + db := New() + defer db.Close() + + // Prepare data + for i := 0; i < count; i++ { + db.Put([]byte(strconv.Itoa(i)), nil) + } + + b.ResetTimer() + b.ReportAllocs() + + // Create batch and delete range + batch := db.NewBatch() + batch.DeleteRange([]byte("0"), []byte("999999999")) + batch.Write() + } + + b.Run("BatchDeleteRange100", func(b *testing.B) { + benchBatchDeleteRange(b, 100) + }) + b.Run("BatchDeleteRange1k", func(b *testing.B) { + benchBatchDeleteRange(b, 1000) + }) + b.Run("BatchDeleteRange10k", func(b *testing.B) { + benchBatchDeleteRange(b, 10000) + }) + }) + + b.Run("BatchMixedOps", func(b *testing.B) { + benchBatchMixedOps := func(b *testing.B, count int) { + db := New() + defer db.Close() + + // Prepare initial data + for i := 0; i < count; i++ { + db.Put([]byte(strconv.Itoa(i)), []byte("val")) + } + + b.ResetTimer() + b.ReportAllocs() + + // Create batch with mixed operations + batch := db.NewBatch() + + // Add some new keys + for i := 0; i < count/10; i++ { + batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val")) + } + + // Delete some individual keys + for i := 0; i < count/20; i++ { + batch.Delete([]byte(strconv.Itoa(i*2))) + } + + // Delete range of keys + rangeStart := count / 2 + rangeEnd := count * 3 / 4 + batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd))) + + // Write the batch + batch.Write() + } + + b.Run("BatchMixedOps100", func(b *testing.B) { + benchBatchMixedOps(b, 100) + }) + b.Run("BatchMixedOps1k", func(b *testing.B) { + benchBatchMixedOps(b, 1000) + }) + b.Run("BatchMixedOps10k", func(b *testing.B) { + benchBatchMixedOps(b, 10000) + }) + }) } func iterateKeys(it ethdb.Iterator) []string { diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 223d01aff6..173a0660db 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -461,6 +461,68 @@ func (b *batch) Delete(key []byte) error { return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +// Note that this is a fallback implementation as leveldb does not natively +// support range deletion in batches. It iterates through the database to find +// keys in the range and adds them to the batch for deletion. +func (b *batch) DeleteRange(start, end []byte) error { + // Special case: empty range + if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) { + return nil + } + + // Special case: delete all keys less than end + if len(start) == 0 { + it := b.db.NewIterator(nil, nil) + defer it.Release() + + for it.Next() { + key := it.Key() + if bytes.Compare(key, end) < 0 { + b.b.Delete(key) + b.size += len(key) + } + } + return it.Error() + } + + // Special case: delete all keys greater than or equal to start + if len(end) == 0 { + it := b.db.NewIterator(nil, nil) + defer it.Release() + + for it.Next() { + key := it.Key() + if bytes.Compare(key, start) >= 0 { + b.b.Delete(key) + b.size += len(key) + } + } + return it.Error() + } + + // Create an iterator to scan through the keys in the range + it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) + defer it.Release() + + var count int + for it.Next() { + count++ + key := it.Key() + if count > 10000 { // should not block for more than a second + return ethdb.ErrTooManyKeys + } + // Add this key to the batch for deletion + b.b.Delete(key) + b.size += len(key) + } + + if err := it.Error(); err != nil { + return err + } + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -506,6 +568,15 @@ func (r *replayer) Delete(key []byte) { r.failure = r.writer.Delete(key) } +// DeleteRange removes all keys in the range [start, end) from the key-value data store. +func (r *replayer) DeleteRange(start, end []byte) { + // If the replay already failed, stop executing ops + if r.failure != nil { + return + } + r.failure = r.writer.DeleteRange(start, end) +} + // bytesPrefixRange returns key range that satisfy // - the given prefix, and // - the given seek position diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index f56727cf4a..436c045f15 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -20,6 +20,7 @@ package memorydb import ( "errors" "sort" + "strconv" "strings" "sync" @@ -130,8 +131,36 @@ func (db *Database) DeleteRange(start, end []byte) error { return errMemorydbClosed } + startStr := string(start) + endStr := string(end) + + if startStr == "" && endStr == "" { + return nil // Empty range, do nothing + } + + if startStr == "" { + // Delete all keys less than end + for key := range db.db { + if key < endStr { + delete(db.db, key) + } + } + return nil + } + + if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) { + // Delete all keys greater than or equal to start + for key := range db.db { + if key >= startStr { + delete(db.db, key) + } + } + return nil + } + + // Normal case: delete keys in range [start, end) for key := range db.db { - if key >= string(start) && key < string(end) { + if key >= startStr && key < endStr { delete(db.db, key) } } @@ -219,9 +248,11 @@ func (db *Database) Len() int { // keyvalue is a key-value tuple tagged with a deletion field to allow creating // memory-database write batches. type keyvalue struct { - key string - value []byte - delete bool + key string + value []byte + delete bool + rangeFrom string + rangeTo string } // batch is a write-only memory batch that commits changes to its host @@ -234,18 +265,29 @@ type batch struct { // Put inserts the given value into the batch for later committing. func (b *batch) Put(key, value []byte) error { - b.writes = append(b.writes, keyvalue{string(key), common.CopyBytes(value), false}) + b.writes = append(b.writes, keyvalue{key: string(key), value: common.CopyBytes(value)}) b.size += len(key) + len(value) return nil } // Delete inserts the key removal into the batch for later committing. func (b *batch) Delete(key []byte) error { - b.writes = append(b.writes, keyvalue{string(key), nil, true}) + b.writes = append(b.writes, keyvalue{key: string(key), delete: true}) b.size += len(key) return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +func (b *batch) DeleteRange(start, end []byte) error { + b.writes = append(b.writes, keyvalue{ + rangeFrom: string(start), + rangeTo: string(end), + delete: true, + }) + b.size += len(start) + len(end) + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -261,7 +303,41 @@ func (b *batch) Write() error { } for _, keyvalue := range b.writes { if keyvalue.delete { - delete(b.db.db, keyvalue.key) + if keyvalue.key != "" { + // Single key deletion + delete(b.db.db, keyvalue.key) + } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + // Range deletion (inclusive of start, exclusive of end) + // Handle special cases for empty ranges + if keyvalue.rangeFrom == keyvalue.rangeTo { + // Empty range, do nothing + continue + } + + // Check if both are numeric strings for consistent comparison + startNum, startErr := strconv.Atoi(keyvalue.rangeFrom) + endNum, endErr := strconv.Atoi(keyvalue.rangeTo) + + // If both are valid integers, use numeric comparison + if startErr == nil && endErr == nil { + for key := range b.db.db { + if keyNum, err := strconv.Atoi(key); err == nil { + if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) { + delete(b.db.db, key) + } + } + } + } else { + // Use string comparison + for key := range b.db.db { + // Handle open ranges + if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && + (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) { + delete(b.db.db, key) + } + } + } + } continue } b.db.db[keyvalue.key] = keyvalue.value @@ -279,8 +355,16 @@ func (b *batch) Reset() { func (b *batch) Replay(w ethdb.KeyValueWriter) error { for _, keyvalue := range b.writes { if keyvalue.delete { - if err := w.Delete([]byte(keyvalue.key)); err != nil { - return err + if keyvalue.key != "" { + // Single key deletion + if err := w.Delete([]byte(keyvalue.key)); err != nil { + return err + } + } else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" { + // Range deletion + if err := w.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil { + return err + } } continue } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 5c851af910..3a026508de 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -619,6 +619,16 @@ func (b *batch) Delete(key []byte) error { return nil } +// DeleteRange removes all keys in the range [start, end) from the batch for later committing. +func (b *batch) DeleteRange(start, end []byte) error { + if err := b.b.DeleteRange(start, end, nil); err != nil { + return err + } + // Approximate size impact - just the keys + b.size += len(start) + len(end) + return nil +} + // ValueSize retrieves the amount of data queued up for writing. func (b *batch) ValueSize() int { return b.size @@ -658,6 +668,11 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { if err = w.Delete(k); err != nil { return err } + } else if kind == pebble.InternalKeyKindRangeDelete { + // For range deletion, k is the start key and v is the end key + if err = w.DeleteRange(k, v); err != nil { + return err + } } else { return fmt.Errorf("unhandled operation, keytype: %v", kind) }