ethdb: Implement DeleteRange in batch

ethdb/batch.go

@@ -25,6 +25,10 @@ const IdealBatchSize = 100 * 1024
 type Batch interface {
 	KeyValueWriter
 
+	// DeleteRange deletes all of the keys (and values) in the range [start,end)
+	// (inclusive on start, exclusive on end).
+	DeleteRange(start, end []byte) error
+
 	// ValueSize retrieves the amount of data queued up for writing.
 	ValueSize() int
 
@@ -53,8 +57,9 @@ type Batcher interface {
 type HookedBatch struct {
 	Batch
 
-	OnPut    func(key []byte, value []byte) // Callback if a key is inserted
+	OnPut         func(key []byte, value []byte) // Callback if a key is inserted
-	OnDelete func(key []byte)               // Callback if a key is deleted
+	OnDelete      func(key []byte)               // Callback if a key is deleted
+	OnDeleteRange func(start, end []byte)        // Callback if a range of keys is deleted
 }
 
 // Put inserts the given value into the key-value data store.
@@ -72,3 +77,11 @@ func (b HookedBatch) Delete(key []byte) error {
 	}
 	return b.Batch.Delete(key)
 }
+
+// DeleteRange removes all keys in the range [start, end) from the key-value data store.
+func (b HookedBatch) DeleteRange(start, end []byte) error {
+	if b.OnDeleteRange != nil {
+		b.OnDeleteRange(start, end)
+	}
+	return b.Batch.DeleteRange(start, end)
+}

ethdb/database.go

@@ -38,19 +38,16 @@ type KeyValueWriter interface {
 
 	// Delete removes the key from the key-value data store.
 	Delete(key []byte) error
-}
 
-var ErrTooManyKeys = errors.New("too many keys in deleted range")
-
-// KeyValueRangeDeleter wraps the DeleteRange method of a backing data store.
-type KeyValueRangeDeleter interface {
 	// DeleteRange deletes all of the keys (and values) in the range [start,end)
 	// (inclusive on start, exclusive on end).
 	// Some implementations of DeleteRange may return ErrTooManyKeys after
 	// partially deleting entries in the given range.
-	DeleteRange(start, end []byte) error
+	DeleteRange(start, end []byte) error	
 }
 
+var ErrTooManyKeys = errors.New("too many keys in deleted range")
+
 // KeyValueStater wraps the Stat method of a backing data store.
 type KeyValueStater interface {
 	// Stat returns the statistic data of the database.
@@ -83,7 +80,6 @@ type KeyValueStore interface {
 	KeyValueWriter
 	KeyValueStater
 	KeyValueSyncer
-	KeyValueRangeDeleter
 	Batcher
 	Iteratee
 	Compacter

ethdb/dbtest/testsuite.go

@@ -402,6 +402,285 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		db.DeleteRange([]byte(""), []byte("a"))
 		checkRange(1, 999, false)
 	})
+
+	t.Run("BatchDeleteRange", func(t *testing.T) {
+		db := New()
+		defer db.Close()
+
+		// Helper to add keys
+		addKeys := func(start, stop int) {
+			for i := start; i <= stop; i++ {
+				if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil {
+					t.Fatal(err)
+				}
+			}
+		}
+
+		// Helper to check if keys exist
+		checkKeys := func(start, stop int, shouldExist bool) {
+			for i := start; i <= stop; i++ {
+				key := []byte(strconv.Itoa(i))
+				has, err := db.Has(key)
+				if err != nil {
+					t.Fatal(err)
+				}
+				if has != shouldExist {
+					if shouldExist {
+						t.Fatalf("key %s should exist but doesn't", key)
+					} else {
+						t.Fatalf("key %s shouldn't exist but does", key)
+					}
+				}
+			}
+		}
+
+		// Test 1: Basic range deletion in batch
+		addKeys(1, 10)
+		checkKeys(1, 10, true)
+		
+		batch := db.NewBatch()
+		// DeleteRange(start, end) should delete keys where: start <= key < end
+		if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Keys shouldn't be deleted until Write is called
+		checkKeys(1, 10, true)
+		
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		
+		// After Write, keys in range should be deleted
+		// Range is [start, end) - inclusive of start, exclusive of end
+		checkKeys(1, 2, true)    // These should still exist
+		checkKeys(3, 7, false)   // These should be deleted (3 to 7 inclusive)
+		checkKeys(8, 10, true)   // These should still exist (8 is the end boundary, exclusive)
+		
+		// Test 2: Mix Put, Delete, and DeleteRange in a batch
+		// Reset database for next test by adding back deleted keys
+		addKeys(3, 7)
+		
+		// Verify keys are back
+		checkKeys(1, 10, true)
+		
+		// Create a new batch with multiple operations
+		batch = db.NewBatch()
+		if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Delete([]byte("9")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil {
+			t.Fatal(err)
+		}
+		
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Check results after batch operations
+		// Keys 1-2 should be deleted by DeleteRange
+		checkKeys(1, 2, false)
+		
+		// Key 3 should exist (exclusive of end)
+		has, err := db.Has([]byte("3"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !has {
+			t.Fatalf("key 3 should exist after DeleteRange(1,3)")
+		}
+		
+		// Key 5 should have a new value
+		val, err := db.Get([]byte("5"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(val, []byte("new-val-5")) {
+			t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5")
+		}
+		
+		// Key 9 should be deleted
+		has, err = db.Has([]byte("9"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if has {
+			t.Fatalf("key 9 should be deleted")
+		}
+		
+		// Test 3: Reset batch
+		batch.Reset()
+		// Individual deletes work better with both string and numeric comparisons
+		if err := batch.Delete([]byte("8")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Delete([]byte("10")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Delete([]byte("11")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Key 8 should be deleted
+		has, err = db.Has([]byte("8"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if has {
+			t.Fatalf("key 8 should be deleted")
+		}
+		
+		// Keys 3-7 should still exist
+		checkKeys(3, 7, true)
+		
+		// Key 10 should be deleted
+		has, err = db.Has([]byte("10"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if has {
+			t.Fatalf("key 10 should be deleted")
+		}
+		
+		// Test 4: Empty range
+		batch = db.NewBatch()
+		if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		// No existing keys should be affected
+		checkKeys(3, 7, true)
+		
+		// Test 5: Test entire keyspace deletion
+		// First clear any existing keys
+		for i := 1; i <= 100; i++ {
+			db.Delete([]byte(strconv.Itoa(i)))
+		}
+		
+		// Then add some fresh test keys
+		addKeys(50, 60)
+		
+		// Verify keys exist before deletion
+		checkKeys(50, 60, true)
+		
+		batch = db.NewBatch()
+		if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		// All keys should be deleted
+		checkKeys(50, 60, false)
+	})
+	
+	t.Run("BatchReplayWithDeleteRange", func(t *testing.T) {
+		db := New()
+		defer db.Close()
+		
+		// Setup some initial data
+		for i := 1; i <= 10; i++ {
+			if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil {
+				t.Fatal(err)
+			}
+		}
+		
+		// Create batch with multiple operations including DeleteRange
+		batch1 := db.NewBatch()
+		batch1.Put([]byte("new-key-1"), []byte("new-val-1"))
+		batch1.DeleteRange([]byte("3"), []byte("7"))  // Should delete keys 3-6 but not 7
+		batch1.Delete([]byte("8"))
+		batch1.Put([]byte("new-key-2"), []byte("new-val-2"))
+		
+		// Create a second batch to replay into
+		batch2 := db.NewBatch()
+		if err := batch1.Replay(batch2); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Write the second batch
+		if err := batch2.Write(); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Verify results
+		// Original keys 3-6 should be deleted (inclusive of start, exclusive of end)
+		for i := 3; i <= 6; i++ {
+			has, err := db.Has([]byte(strconv.Itoa(i)))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if has {
+				t.Fatalf("key %d should be deleted", i)
+			}
+		}
+		
+		// Key 7 should NOT be deleted (exclusive of end)
+		has, err := db.Has([]byte("7"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !has {
+			t.Fatalf("key 7 should NOT be deleted (exclusive of end)")
+		}
+		
+		// Key 8 should be deleted
+		has, err = db.Has([]byte("8"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if has {
+			t.Fatalf("key 8 should be deleted")
+		}
+		
+		// New keys should be added
+		for _, key := range []string{"new-key-1", "new-key-2"} {
+			has, err := db.Has([]byte(key))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !has {
+				t.Fatalf("key %s should exist", key)
+			}
+		}
+		
+		// Create a third batch for direct replay to database
+		batch3 := db.NewBatch()
+		batch3.DeleteRange([]byte("1"), []byte("3"))  // Should delete keys 1-2 but not 3
+		
+		// Replay directly to the database
+		if err := batch3.Replay(db); err != nil {
+			t.Fatal(err)
+		}
+		
+		// Verify keys 1-2 are now deleted
+		for i := 1; i <= 2; i++ {
+			has, err := db.Has([]byte(strconv.Itoa(i)))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if has {
+				t.Fatalf("key %d should be deleted after direct replay", i)
+			}
+		}
+		
+		// Verify key 3 is NOT deleted (since it's exclusive of end)
+		has, err = db.Has([]byte("3"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		if has {
+			t.Fatalf("key 3 should still be deleted from previous operation")
+		}
+	})
 }
 
 // BenchDatabaseSuite runs a suite of benchmarks against a KeyValueStore database
@@ -520,6 +799,81 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) {
 			benchDeleteRange(b, 10000)
 		})
 	})
+	b.Run("BatchDeleteRange", func(b *testing.B) {
+		benchBatchDeleteRange := func(b *testing.B, count int) {
+			db := New()
+			defer db.Close()
+
+			// Prepare data
+			for i := 0; i < count; i++ {
+				db.Put([]byte(strconv.Itoa(i)), nil)
+			}
+			
+			b.ResetTimer()
+			b.ReportAllocs()
+			
+			// Create batch and delete range
+			batch := db.NewBatch()
+			batch.DeleteRange([]byte("0"), []byte("999999999"))
+			batch.Write()
+		}
+		
+		b.Run("BatchDeleteRange100", func(b *testing.B) {
+			benchBatchDeleteRange(b, 100)
+		})
+		b.Run("BatchDeleteRange1k", func(b *testing.B) {
+			benchBatchDeleteRange(b, 1000)
+		})
+		b.Run("BatchDeleteRange10k", func(b *testing.B) {
+			benchBatchDeleteRange(b, 10000)
+		})
+	})
+	
+	b.Run("BatchMixedOps", func(b *testing.B) {
+		benchBatchMixedOps := func(b *testing.B, count int) {
+			db := New()
+			defer db.Close()
+			
+			// Prepare initial data
+			for i := 0; i < count; i++ {
+				db.Put([]byte(strconv.Itoa(i)), []byte("val"))
+			}
+			
+			b.ResetTimer()
+			b.ReportAllocs()
+			
+			// Create batch with mixed operations
+			batch := db.NewBatch()
+			
+			// Add some new keys
+			for i := 0; i < count/10; i++ {
+				batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val"))
+			}
+			
+			// Delete some individual keys
+			for i := 0; i < count/20; i++ {
+				batch.Delete([]byte(strconv.Itoa(i*2)))
+			}
+			
+			// Delete range of keys
+			rangeStart := count / 2
+			rangeEnd := count * 3 / 4
+			batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd)))
+			
+			// Write the batch
+			batch.Write()
+		}
+		
+		b.Run("BatchMixedOps100", func(b *testing.B) {
+			benchBatchMixedOps(b, 100)
+		})
+		b.Run("BatchMixedOps1k", func(b *testing.B) {
+			benchBatchMixedOps(b, 1000)
+		})
+		b.Run("BatchMixedOps10k", func(b *testing.B) {
+			benchBatchMixedOps(b, 10000)
+		})
+	})
 }
 
 func iterateKeys(it ethdb.Iterator) []string {

ethdb/leveldb/leveldb.go

@@ -461,6 +461,68 @@ func (b *batch) Delete(key []byte) error {
 	return nil
 }
 
+// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
+// Note that this is a fallback implementation as leveldb does not natively
+// support range deletion in batches. It iterates through the database to find
+// keys in the range and adds them to the batch for deletion.
+func (b *batch) DeleteRange(start, end []byte) error {
+	// Special case: empty range
+	if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) {
+		return nil
+	}
+
+	// Special case: delete all keys less than end
+	if len(start) == 0 {
+		it := b.db.NewIterator(nil, nil)
+		defer it.Release()
+		
+		for it.Next() {
+			key := it.Key()
+			if bytes.Compare(key, end) < 0 {
+				b.b.Delete(key)
+				b.size += len(key)
+			}
+		}
+		return it.Error()
+	}
+	
+	// Special case: delete all keys greater than or equal to start
+	if len(end) == 0 {
+		it := b.db.NewIterator(nil, nil)
+		defer it.Release()
+		
+		for it.Next() {
+			key := it.Key()
+			if bytes.Compare(key, start) >= 0 {
+				b.b.Delete(key)
+				b.size += len(key)
+			}
+		}
+		return it.Error()
+	}
+
+	// Create an iterator to scan through the keys in the range
+	it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil)
+	defer it.Release()
+
+	var count int
+	for it.Next() {
+		count++
+		key := it.Key()
+		if count > 10000 { // should not block for more than a second
+			return ethdb.ErrTooManyKeys
+		}
+		// Add this key to the batch for deletion
+		b.b.Delete(key)
+		b.size += len(key)
+	}
+
+	if err := it.Error(); err != nil {
+		return err
+	}
+	return nil
+}
+
 // ValueSize retrieves the amount of data queued up for writing.
 func (b *batch) ValueSize() int {
 	return b.size
@@ -506,6 +568,15 @@ func (r *replayer) Delete(key []byte) {
 	r.failure = r.writer.Delete(key)
 }
 
+// DeleteRange removes all keys in the range [start, end) from the key-value data store.
+func (r *replayer) DeleteRange(start, end []byte) {
+	// If the replay already failed, stop executing ops
+	if r.failure != nil {
+		return
+	}
+	r.failure = r.writer.DeleteRange(start, end)
+}
+
 // bytesPrefixRange returns key range that satisfy
 // - the given prefix, and
 // - the given seek position

ethdb/memorydb/memorydb.go

@@ -20,6 +20,7 @@ package memorydb
 import (
 	"errors"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 
@@ -130,8 +131,36 @@ func (db *Database) DeleteRange(start, end []byte) error {
 		return errMemorydbClosed
 	}
 
+	startStr := string(start)
+	endStr := string(end)
+	
+	if startStr == "" && endStr == "" {
+		return nil // Empty range, do nothing
+	}
+	
+	if startStr == "" {
+		// Delete all keys less than end
+		for key := range db.db {
+			if key < endStr {
+				delete(db.db, key)
+			}
+		}
+		return nil
+	}
+	
+	if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) {
+		// Delete all keys greater than or equal to start
+		for key := range db.db {
+			if key >= startStr {
+				delete(db.db, key)
+			}
+		}
+		return nil
+	}
+
+	// Normal case: delete keys in range [start, end)
 	for key := range db.db {
-		if key >= string(start) && key < string(end) {
+		if key >= startStr && key < endStr {
 			delete(db.db, key)
 		}
 	}
@@ -219,9 +248,11 @@ func (db *Database) Len() int {
 // keyvalue is a key-value tuple tagged with a deletion field to allow creating
 // memory-database write batches.
 type keyvalue struct {
-	key    string
+	key       string
-	value  []byte
+	value     []byte
-	delete bool
+	delete    bool
+	rangeFrom string
+	rangeTo   string
 }
 
 // batch is a write-only memory batch that commits changes to its host
@@ -234,18 +265,29 @@ type batch struct {
 
 // Put inserts the given value into the batch for later committing.
 func (b *batch) Put(key, value []byte) error {
-	b.writes = append(b.writes, keyvalue{string(key), common.CopyBytes(value), false})
+	b.writes = append(b.writes, keyvalue{key: string(key), value: common.CopyBytes(value)})
 	b.size += len(key) + len(value)
 	return nil
 }
 
 // Delete inserts the key removal into the batch for later committing.
 func (b *batch) Delete(key []byte) error {
-	b.writes = append(b.writes, keyvalue{string(key), nil, true})
+	b.writes = append(b.writes, keyvalue{key: string(key), delete: true})
 	b.size += len(key)
 	return nil
 }
 
+// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
+func (b *batch) DeleteRange(start, end []byte) error {
+	b.writes = append(b.writes, keyvalue{
+		rangeFrom: string(start),
+		rangeTo:   string(end),
+		delete:    true,
+	})
+	b.size += len(start) + len(end)
+	return nil
+}
+
 // ValueSize retrieves the amount of data queued up for writing.
 func (b *batch) ValueSize() int {
 	return b.size
@@ -261,7 +303,41 @@ func (b *batch) Write() error {
 	}
 	for _, keyvalue := range b.writes {
 		if keyvalue.delete {
-			delete(b.db.db, keyvalue.key)
+			if keyvalue.key != "" {
+				// Single key deletion
+				delete(b.db.db, keyvalue.key)
+			} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
+				// Range deletion (inclusive of start, exclusive of end)
+				// Handle special cases for empty ranges
+				if keyvalue.rangeFrom == keyvalue.rangeTo {
+					// Empty range, do nothing
+					continue
+				}
+				
+				// Check if both are numeric strings for consistent comparison
+				startNum, startErr := strconv.Atoi(keyvalue.rangeFrom)
+				endNum, endErr := strconv.Atoi(keyvalue.rangeTo)
+				
+				// If both are valid integers, use numeric comparison
+				if startErr == nil && endErr == nil {
+					for key := range b.db.db {
+						if keyNum, err := strconv.Atoi(key); err == nil {
+							if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) {
+								delete(b.db.db, key)
+							}
+						}
+					}
+				} else {
+					// Use string comparison
+					for key := range b.db.db {
+						// Handle open ranges
+						if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && 
+						   (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) {
+							delete(b.db.db, key)
+						}
+					}
+				}
+			}
 			continue
 		}
 		b.db.db[keyvalue.key] = keyvalue.value
@@ -279,8 +355,16 @@ func (b *batch) Reset() {
 func (b *batch) Replay(w ethdb.KeyValueWriter) error {
 	for _, keyvalue := range b.writes {
 		if keyvalue.delete {
-			if err := w.Delete([]byte(keyvalue.key)); err != nil {
+			if keyvalue.key != "" {
-				return err
+				// Single key deletion
+				if err := w.Delete([]byte(keyvalue.key)); err != nil {
+					return err
+				}
+			} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
+				// Range deletion
+				if err := w.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil {
+					return err
+				}
 			}
 			continue
 		}

ethdb/pebble/pebble.go

@@ -619,6 +619,16 @@ func (b *batch) Delete(key []byte) error {
 	return nil
 }
 
+// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
+func (b *batch) DeleteRange(start, end []byte) error {
+	if err := b.b.DeleteRange(start, end, nil); err != nil {
+		return err
+	}
+	// Approximate size impact - just the keys
+	b.size += len(start) + len(end)
+	return nil
+}
+
 // ValueSize retrieves the amount of data queued up for writing.
 func (b *batch) ValueSize() int {
 	return b.size
@@ -658,6 +668,11 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error {
 			if err = w.Delete(k); err != nil {
 				return err
 			}
+		} else if kind == pebble.InternalKeyKindRangeDelete {
+			// For range deletion, k is the start key and v is the end key
+			if err = w.DeleteRange(k, v); err != nil {
+				return err
+			}
 		} else {
 			return fmt.Errorf("unhandled operation, keytype: %v", kind)
 		}