ethdb: simlify the range deletion

ethdb/database.go

@@ -46,9 +46,14 @@ var ErrTooManyKeys = errors.New("too many keys in deleted range")
 type KeyValueRangeDeleter interface {
 	// DeleteRange deletes all of the keys (and values) in the range [start,end)
 	// (inclusive on start, exclusive on end).
+	//
+	// A nil start is treated as a key before all keys in the data store; a nil
+	// end is treated as a key after all keys in the data store. If both is nil
+	// then the entire data store will be purged.
+	//
 	// Some implementations of DeleteRange may return ErrTooManyKeys after
 	// partially deleting entries in the given range.
-	DeleteRange(start, end []byte) error	
+	DeleteRange(start, end []byte) error
 }
 
 // KeyValueStater wraps the Stat method of a backing data store.

ethdb/dbtest/testsuite.go

@@ -401,6 +401,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 
 		db.DeleteRange([]byte(""), []byte("a"))
 		checkRange(1, 999, false)
+
+		addRange(1, 999)
+		db.DeleteRange(nil, nil)
+		checkRange(1, 999, false)
 	})
 
 	t.Run("BatchDeleteRange", func(t *testing.T) {
@@ -437,33 +441,39 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		// Test 1: Basic range deletion in batch
 		addKeys(1, 10)
 		checkKeys(1, 10, true)
-		
+
 		batch := db.NewBatch()
-		// DeleteRange(start, end) should delete keys where: start <= key < end
 		if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil {
 			t.Fatal(err)
 		}
-		
 		// Keys shouldn't be deleted until Write is called
 		checkKeys(1, 10, true)
-		
+
 		if err := batch.Write(); err != nil {
 			t.Fatal(err)
 		}
-		
 		// After Write, keys in range should be deleted
 		// Range is [start, end) - inclusive of start, exclusive of end
-		checkKeys(1, 2, true)    // These should still exist
+		checkKeys(1, 2, true)  // These should still exist
-		checkKeys(3, 7, false)   // These should be deleted (3 to 7 inclusive)
+		checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive)
-		checkKeys(8, 10, true)   // These should still exist (8 is the end boundary, exclusive)
+		checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive)
-		
+
-		// Test 2: Mix Put, Delete, and DeleteRange in a batch
+		// Test 2: Delete range with special markers
-		// Reset database for next test by adding back deleted keys
 		addKeys(3, 7)
-		
+		batch = db.NewBatch()
-		// Verify keys are back
+		if err := batch.DeleteRange(nil, nil); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		checkKeys(1, 10, false)
+
+		// Test 3: Mix Put, Delete, and DeleteRange in a batch
+		// Reset database for next test by adding back deleted keys
+		addKeys(1, 10)
 		checkKeys(1, 10, true)
-		
+
 		// Create a new batch with multiple operations
 		batch = db.NewBatch()
 		if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil {
@@ -475,15 +485,13 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil {
 			t.Fatal(err)
 		}
-		
 		if err := batch.Write(); err != nil {
 			t.Fatal(err)
 		}
-		
 		// Check results after batch operations
 		// Keys 1-2 should be deleted by DeleteRange
 		checkKeys(1, 2, false)
-		
+
 		// Key 3 should exist (exclusive of end)
 		has, err := db.Has([]byte("3"))
 		if err != nil {
@@ -492,7 +500,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if !has {
 			t.Fatalf("key 3 should exist after DeleteRange(1,3)")
 		}
-		
+
 		// Key 5 should have a new value
 		val, err := db.Get([]byte("5"))
 		if err != nil {
@@ -501,7 +509,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if !bytes.Equal(val, []byte("new-val-5")) {
 			t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5")
 		}
-		
+
 		// Key 9 should be deleted
 		has, err = db.Has([]byte("9"))
 		if err != nil {
@@ -510,8 +518,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if has {
 			t.Fatalf("key 9 should be deleted")
 		}
-		
+
-		// Test 3: Reset batch
+		// Test 4: Reset batch
 		batch.Reset()
 		// Individual deletes work better with both string and numeric comparisons
 		if err := batch.Delete([]byte("8")); err != nil {
@@ -526,7 +534,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if err := batch.Write(); err != nil {
 			t.Fatal(err)
 		}
-		
+
 		// Key 8 should be deleted
 		has, err = db.Has([]byte("8"))
 		if err != nil {
@@ -535,10 +543,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if has {
 			t.Fatalf("key 8 should be deleted")
 		}
-		
+
 		// Keys 3-7 should still exist
 		checkKeys(3, 7, true)
-		
+
 		// Key 10 should be deleted
 		has, err = db.Has([]byte("10"))
 		if err != nil {
@@ -547,8 +555,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if has {
 			t.Fatalf("key 10 should be deleted")
 		}
-		
+
-		// Test 4: Empty range
+		// Test 5: Empty range
 		batch = db.NewBatch()
 		if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil {
 			t.Fatal(err)
@@ -558,19 +566,19 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		}
 		// No existing keys should be affected
 		checkKeys(3, 7, true)
-		
+
-		// Test 5: Test entire keyspace deletion
+		// Test 6: Test entire keyspace deletion
 		// First clear any existing keys
 		for i := 1; i <= 100; i++ {
 			db.Delete([]byte(strconv.Itoa(i)))
 		}
-		
+
 		// Then add some fresh test keys
 		addKeys(50, 60)
-		
+
 		// Verify keys exist before deletion
 		checkKeys(50, 60, true)
-		
+
 		batch = db.NewBatch()
 		if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil {
 			t.Fatal(err)
@@ -580,37 +588,52 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		}
 		// All keys should be deleted
 		checkKeys(50, 60, false)
+
+		// Test 7: overlapping range deletion
+		addKeys(50, 60)
+		batch = db.NewBatch()
+		if err := batch.DeleteRange([]byte("50"), []byte("55")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.DeleteRange([]byte("52"), []byte("58")); err != nil {
+			t.Fatal(err)
+		}
+		if err := batch.Write(); err != nil {
+			t.Fatal(err)
+		}
+		checkKeys(50, 57, false)
+		checkKeys(58, 60, true)
 	})
-	
+
 	t.Run("BatchReplayWithDeleteRange", func(t *testing.T) {
 		db := New()
 		defer db.Close()
-		
+
 		// Setup some initial data
 		for i := 1; i <= 10; i++ {
 			if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil {
 				t.Fatal(err)
 			}
 		}
-		
+
 		// Create batch with multiple operations including DeleteRange
 		batch1 := db.NewBatch()
 		batch1.Put([]byte("new-key-1"), []byte("new-val-1"))
-		batch1.DeleteRange([]byte("3"), []byte("7"))  // Should delete keys 3-6 but not 7
+		batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7
 		batch1.Delete([]byte("8"))
 		batch1.Put([]byte("new-key-2"), []byte("new-val-2"))
-		
+
 		// Create a second batch to replay into
 		batch2 := db.NewBatch()
 		if err := batch1.Replay(batch2); err != nil {
 			t.Fatal(err)
 		}
-		
+
 		// Write the second batch
 		if err := batch2.Write(); err != nil {
 			t.Fatal(err)
 		}
-		
+
 		// Verify results
 		// Original keys 3-6 should be deleted (inclusive of start, exclusive of end)
 		for i := 3; i <= 6; i++ {
@@ -622,7 +645,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 				t.Fatalf("key %d should be deleted", i)
 			}
 		}
-		
+
 		// Key 7 should NOT be deleted (exclusive of end)
 		has, err := db.Has([]byte("7"))
 		if err != nil {
@@ -631,7 +654,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if !has {
 			t.Fatalf("key 7 should NOT be deleted (exclusive of end)")
 		}
-		
+
 		// Key 8 should be deleted
 		has, err = db.Has([]byte("8"))
 		if err != nil {
@@ -640,7 +663,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 		if has {
 			t.Fatalf("key 8 should be deleted")
 		}
-		
+
 		// New keys should be added
 		for _, key := range []string{"new-key-1", "new-key-2"} {
 			has, err := db.Has([]byte(key))
@@ -651,16 +674,16 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 				t.Fatalf("key %s should exist", key)
 			}
 		}
-		
+
 		// Create a third batch for direct replay to database
 		batch3 := db.NewBatch()
-		batch3.DeleteRange([]byte("1"), []byte("3"))  // Should delete keys 1-2 but not 3
+		batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3
-		
+
 		// Replay directly to the database
 		if err := batch3.Replay(db); err != nil {
 			t.Fatal(err)
 		}
-		
+
 		// Verify keys 1-2 are now deleted
 		for i := 1; i <= 2; i++ {
 			has, err := db.Has([]byte(strconv.Itoa(i)))
@@ -671,7 +694,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
 				t.Fatalf("key %d should be deleted after direct replay", i)
 			}
 		}
-		
+
 		// Verify key 3 is NOT deleted (since it's exclusive of end)
 		has, err = db.Has([]byte("3"))
 		if err != nil {
@@ -808,16 +831,16 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) {
 			for i := 0; i < count; i++ {
 				db.Put([]byte(strconv.Itoa(i)), nil)
 			}
-			
+
 			b.ResetTimer()
 			b.ReportAllocs()
-			
+
 			// Create batch and delete range
 			batch := db.NewBatch()
 			batch.DeleteRange([]byte("0"), []byte("999999999"))
 			batch.Write()
 		}
-		
+
 		b.Run("BatchDeleteRange100", func(b *testing.B) {
 			benchBatchDeleteRange(b, 100)
 		})
@@ -828,42 +851,42 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) {
 			benchBatchDeleteRange(b, 10000)
 		})
 	})
-	
+
 	b.Run("BatchMixedOps", func(b *testing.B) {
 		benchBatchMixedOps := func(b *testing.B, count int) {
 			db := New()
 			defer db.Close()
-			
+
 			// Prepare initial data
 			for i := 0; i < count; i++ {
 				db.Put([]byte(strconv.Itoa(i)), []byte("val"))
 			}
-			
+
 			b.ResetTimer()
 			b.ReportAllocs()
-			
+
 			// Create batch with mixed operations
 			batch := db.NewBatch()
-			
+
 			// Add some new keys
 			for i := 0; i < count/10; i++ {
 				batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val"))
 			}
-			
+
 			// Delete some individual keys
 			for i := 0; i < count/20; i++ {
-				batch.Delete([]byte(strconv.Itoa(i*2)))
+				batch.Delete([]byte(strconv.Itoa(i * 2)))
 			}
-			
+
 			// Delete range of keys
 			rangeStart := count / 2
 			rangeEnd := count * 3 / 4
 			batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd)))
-			
+
 			// Write the batch
 			batch.Write()
 		}
-		
+
 		b.Run("BatchMixedOps100", func(b *testing.B) {
 			benchBatchMixedOps(b, 100)
 		})

ethdb/leveldb/leveldb.go

@@ -220,7 +220,7 @@ func (db *Database) DeleteRange(start, end []byte) error {
 	defer it.Release()
 
 	var count int
-	for it.Next() && bytes.Compare(end, it.Key()) > 0 {
+	for it.Next() && (end == nil || bytes.Compare(end, it.Key()) > 0) {
 		count++
 		if count > 10000 { // should not block for more than a second
 			if err := batch.Write(); err != nil {
@@ -461,48 +461,19 @@ func (b *batch) Delete(key []byte) error {
 	return nil
 }
 
-// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
+// DeleteRange removes all keys in the range [start, end) from the batch for
+// later committing, inclusive on start, exclusive on end.
+//
 // Note that this is a fallback implementation as leveldb does not natively
 // support range deletion in batches. It iterates through the database to find
 // keys in the range and adds them to the batch for deletion.
 func (b *batch) DeleteRange(start, end []byte) error {
-	// Special case: empty range
-	if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) {
-		return nil
-	}
-
-	// Special case: delete all keys less than end
-	if len(start) == 0 {
-		it := b.db.NewIterator(nil, nil)
-		defer it.Release()
-		
-		for it.Next() {
-			key := it.Key()
-			if bytes.Compare(key, end) < 0 {
-				b.b.Delete(key)
-				b.size += len(key)
-			}
-		}
-		return it.Error()
-	}
-	
-	// Special case: delete all keys greater than or equal to start
-	if len(end) == 0 {
-		it := b.db.NewIterator(nil, nil)
-		defer it.Release()
-		
-		for it.Next() {
-			key := it.Key()
-			if bytes.Compare(key, start) >= 0 {
-				b.b.Delete(key)
-				b.size += len(key)
-			}
-		}
-		return it.Error()
-	}
-
 	// Create an iterator to scan through the keys in the range
-	it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil)
+	slice := &util.Range{
+		Start: start, // If nil, it represents the key before all keys
+		Limit: end,   // If nil, it represents the key after all keys
+	}
+	it := b.db.NewIterator(slice, nil)
 	defer it.Release()
 
 	var count int
@@ -516,7 +487,6 @@ func (b *batch) DeleteRange(start, end []byte) error {
 		b.b.Delete(key)
 		b.size += len(key)
 	}
-
 	if err := it.Error(); err != nil {
 		return err
 	}

ethdb/memorydb/memorydb.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -124,46 +123,24 @@ func (db *Database) Delete(key []byte) error {
 }
 
 // DeleteRange deletes all of the keys (and values) in the range [start,end)
-// (inclusive on start, exclusive on end).
+// (inclusive on start, exclusive on end). If the start is nil, it represents
+// the key before all keys; if the end is nil, it represents the key after
+// all keys.
 func (db *Database) DeleteRange(start, end []byte) error {
 	db.lock.Lock()
 	defer db.lock.Unlock()
+
 	if db.db == nil {
 		return errMemorydbClosed
 	}
-
-	startStr := string(start)
-	endStr := string(end)
-	
-	if startStr == "" && endStr == "" {
-		return nil // Empty range, do nothing
-	}
-	
-	if startStr == "" {
-		// Delete all keys less than end
-		for key := range db.db {
-			if key < endStr {
-				delete(db.db, key)
-			}
-		}
-		return nil
-	}
-	
-	if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) {
-		// Delete all keys greater than or equal to start
-		for key := range db.db {
-			if key >= startStr {
-				delete(db.db, key)
-			}
-		}
-		return nil
-	}
-
-	// Normal case: delete keys in range [start, end)
 	for key := range db.db {
-		if key >= startStr && key < endStr {
+		if start != nil && key < string(start) {
-			delete(db.db, key)
+			continue
 		}
+		if end != nil && key >= string(end) {
+			continue
+		}
+		delete(db.db, key)
 	}
 	return nil
 }
@@ -249,11 +226,12 @@ func (db *Database) Len() int {
 // keyvalue is a key-value tuple tagged with a deletion field to allow creating
 // memory-database write batches.
 type keyvalue struct {
-	key       string
+	key    string
-	value     []byte
+	value  []byte
-	delete    bool
+	delete bool
-	rangeFrom string
+
-	rangeTo   string
+	rangeFrom []byte
+	rangeTo   []byte
 }
 
 // batch is a write-only memory batch that commits changes to its host
@@ -281,8 +259,8 @@ func (b *batch) Delete(key []byte) error {
 // DeleteRange removes all keys in the range [start, end) from the batch for later committing.
 func (b *batch) DeleteRange(start, end []byte) error {
 	b.writes = append(b.writes, keyvalue{
-		rangeFrom: string(start),
+		rangeFrom: start,
-		rangeTo:   string(end),
+		rangeTo:   end,
 		delete:    true,
 	})
 	b.size += len(start) + len(end)
@@ -302,46 +280,26 @@ func (b *batch) Write() error {
 	if b.db.db == nil {
 		return errMemorydbClosed
 	}
-	for _, keyvalue := range b.writes {
+	for _, entry := range b.writes {
-		if keyvalue.delete {
+		if entry.delete {
-			if keyvalue.key != "" {
+			if entry.key != "" {
 				// Single key deletion
-				delete(b.db.db, keyvalue.key)
+				delete(b.db.db, entry.key)
-			} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
+			} else {
 				// Range deletion (inclusive of start, exclusive of end)
-				// Handle special cases for empty ranges
+				for key := range b.db.db {
-				if keyvalue.rangeFrom == keyvalue.rangeTo {
+					if entry.rangeFrom != nil && key < string(entry.rangeFrom) {
-					// Empty range, do nothing
+						continue
-					continue
-				}
-				
-				// Check if both are numeric strings for consistent comparison
-				startNum, startErr := strconv.Atoi(keyvalue.rangeFrom)
-				endNum, endErr := strconv.Atoi(keyvalue.rangeTo)
-				
-				// If both are valid integers, use numeric comparison
-				if startErr == nil && endErr == nil {
-					for key := range b.db.db {
-						if keyNum, err := strconv.Atoi(key); err == nil {
-							if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) {
-								delete(b.db.db, key)
-							}
-						}
 					}
-				} else {
+					if entry.rangeTo != nil && key >= string(entry.rangeTo) {
-					// Use string comparison
+						continue
-					for key := range b.db.db {
-						// Handle open ranges
-						if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) && 
-						   (keyvalue.rangeTo == "" || key < keyvalue.rangeTo) {
-							delete(b.db.db, key)
-						}
 					}
+					delete(b.db.db, key)
 				}
 			}
 			continue
 		}
-		b.db.db[keyvalue.key] = keyvalue.value
+		b.db.db[entry.key] = entry.value
 	}
 	return nil
 }
@@ -354,17 +312,17 @@ func (b *batch) Reset() {
 
 // Replay replays the batch contents.
 func (b *batch) Replay(w ethdb.KeyValueWriter) error {
-	for _, keyvalue := range b.writes {
+	for _, entry := range b.writes {
-		if keyvalue.delete {
+		if entry.delete {
-			if keyvalue.key != "" {
+			if entry.key != "" {
 				// Single key deletion
-				if err := w.Delete([]byte(keyvalue.key)); err != nil {
+				if err := w.Delete([]byte(entry.key)); err != nil {
 					return err
 				}
-			} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
+			} else {
 				// Range deletion
 				if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok {
-					if err := rangeDeleter.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil {
+					if err := rangeDeleter.DeleteRange(entry.rangeFrom, entry.rangeTo); err != nil {
 						return err
 					}
 				} else {
@@ -373,7 +331,7 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error {
 			}
 			continue
 		}
-		if err := w.Put([]byte(keyvalue.key), keyvalue.value); err != nil {
+		if err := w.Put([]byte(entry.key), entry.value); err != nil {
 			return err
 		}
 	}

ethdb/pebble/pebble.go

@@ -403,9 +403,16 @@ func (d *Database) Delete(key []byte) error {
 func (d *Database) DeleteRange(start, end []byte) error {
 	d.quitLock.RLock()
 	defer d.quitLock.RUnlock()
+
 	if d.closed {
 		return pebble.ErrClosed
 	}
+	// There is no special flag to represent the end of key range
+	// in pebble(nil in leveldb). Use an ugly hack to construct a
+	// large key to represent it.
+	if end == nil {
+		end = bytes.Repeat([]byte{0xff}, 32)
+	}
 	return d.db.DeleteRange(start, end, d.writeOptions)
 }
 
@@ -619,8 +626,15 @@ func (b *batch) Delete(key []byte) error {
 	return nil
 }
 
-// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
+// DeleteRange removes all keys in the range [start, end) from the batch for
+// later committing, inclusive on start, exclusive on end.
 func (b *batch) DeleteRange(start, end []byte) error {
+	// There is no special flag to represent the end of key range
+	// in pebble(nil in leveldb). Use an ugly hack to construct a
+	// large key to represent it.
+	if end == nil {
+		end = bytes.Repeat([]byte{0xff}, 32)
+	}
 	if err := b.b.DeleteRange(start, end, nil); err != nil {
 		return err
 	}