forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-24 21:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

ethdb: simlify the range deletion

Browse source
This commit is contained in:
Gary Rong 2025-06-18 11:29:22 +08:00
parent 6f76fec4c5
commit cabe2dafde
5 changed files with 149 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
ethdb/database.go
View file

@ -46,9 +46,14 @@ var ErrTooManyKeys = errors.New("too many keys in deleted range")
type KeyValueRangeDeleter interface {
// DeleteRange deletes all of the keys (and values) in the range [start,end)
// (inclusive on start, exclusive on end).
//
// A nil start is treated as a key before all keys in the data store; a nil
// end is treated as a key after all keys in the data store. If both is nil
// then the entire data store will be purged.
//
// Some implementations of DeleteRange may return ErrTooManyKeys after
// partially deleting entries in the given range.
DeleteRange(start, end []byte) error
DeleteRange(start, end []byte) error
}
// KeyValueStater wraps the Stat method of a backing data store.

141
ethdb/dbtest/testsuite.go
View file

@ -401,6 +401,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
db.DeleteRange([]byte(""), []byte("a"))
checkRange(1, 999, false)
addRange(1, 999)
db.DeleteRange(nil, nil)
checkRange(1, 999, false)
})
t.Run("BatchDeleteRange", func(t *testing.T) {
@ -437,33 +441,39 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
// Test 1: Basic range deletion in batch
addKeys(1, 10)
checkKeys(1, 10, true)
batch := db.NewBatch()
// DeleteRange(start, end) should delete keys where: start <= key < end
if err := batch.DeleteRange([]byte("3"), []byte("8")); err != nil {
t.Fatal(err)
}
// Keys shouldn't be deleted until Write is called
checkKeys(1, 10, true)
if err := batch.Write(); err != nil {
t.Fatal(err)
}
// After Write, keys in range should be deleted
// Range is [start, end) - inclusive of start, exclusive of end
checkKeys(1, 2, true) // These should still exist
checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive)
checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive)
// Test 2: Mix Put, Delete, and DeleteRange in a batch
// Reset database for next test by adding back deleted keys
checkKeys(1, 2, true) // These should still exist
checkKeys(3, 7, false) // These should be deleted (3 to 7 inclusive)
checkKeys(8, 10, true) // These should still exist (8 is the end boundary, exclusive)
// Test 2: Delete range with special markers
addKeys(3, 7)
// Verify keys are back
batch = db.NewBatch()
if err := batch.DeleteRange(nil, nil); err != nil {
t.Fatal(err)
}
if err := batch.Write(); err != nil {
t.Fatal(err)
}
checkKeys(1, 10, false)
// Test 3: Mix Put, Delete, and DeleteRange in a batch
// Reset database for next test by adding back deleted keys
addKeys(1, 10)
checkKeys(1, 10, true)
// Create a new batch with multiple operations
batch = db.NewBatch()
if err := batch.Put([]byte("5"), []byte("new-val-5")); err != nil {
@ -475,15 +485,13 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if err := batch.DeleteRange([]byte("1"), []byte("3")); err != nil {
t.Fatal(err)
}
if err := batch.Write(); err != nil {
t.Fatal(err)
}
// Check results after batch operations
// Keys 1-2 should be deleted by DeleteRange
checkKeys(1, 2, false)
// Key 3 should exist (exclusive of end)
has, err := db.Has([]byte("3"))
if err != nil {
@ -492,7 +500,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if !has {
t.Fatalf("key 3 should exist after DeleteRange(1,3)")
}
// Key 5 should have a new value
val, err := db.Get([]byte("5"))
if err != nil {
@ -501,7 +509,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if !bytes.Equal(val, []byte("new-val-5")) {
t.Fatalf("key 5 has wrong value: got %s, want %s", val, "new-val-5")
}
// Key 9 should be deleted
has, err = db.Has([]byte("9"))
if err != nil {
@ -510,8 +518,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if has {
t.Fatalf("key 9 should be deleted")
}
// Test 3: Reset batch
// Test 4: Reset batch
batch.Reset()
// Individual deletes work better with both string and numeric comparisons
if err := batch.Delete([]byte("8")); err != nil {
@ -526,7 +534,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if err := batch.Write(); err != nil {
t.Fatal(err)
}
// Key 8 should be deleted
has, err = db.Has([]byte("8"))
if err != nil {
@ -535,10 +543,10 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if has {
t.Fatalf("key 8 should be deleted")
}
// Keys 3-7 should still exist
checkKeys(3, 7, true)
// Key 10 should be deleted
has, err = db.Has([]byte("10"))
if err != nil {
@ -547,8 +555,8 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if has {
t.Fatalf("key 10 should be deleted")
}
// Test 4: Empty range
// Test 5: Empty range
batch = db.NewBatch()
if err := batch.DeleteRange([]byte("100"), []byte("100")); err != nil {
t.Fatal(err)
@ -558,19 +566,19 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
}
// No existing keys should be affected
checkKeys(3, 7, true)
// Test 5: Test entire keyspace deletion
// Test 6: Test entire keyspace deletion
// First clear any existing keys
for i := 1; i <= 100; i++ {
db.Delete([]byte(strconv.Itoa(i)))
}
// Then add some fresh test keys
addKeys(50, 60)
// Verify keys exist before deletion
checkKeys(50, 60, true)
batch = db.NewBatch()
if err := batch.DeleteRange([]byte(""), []byte("z")); err != nil {
t.Fatal(err)
@ -580,37 +588,52 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
}
// All keys should be deleted
checkKeys(50, 60, false)
// Test 7: overlapping range deletion
addKeys(50, 60)
batch = db.NewBatch()
if err := batch.DeleteRange([]byte("50"), []byte("55")); err != nil {
t.Fatal(err)
}
if err := batch.DeleteRange([]byte("52"), []byte("58")); err != nil {
t.Fatal(err)
}
if err := batch.Write(); err != nil {
t.Fatal(err)
}
checkKeys(50, 57, false)
checkKeys(58, 60, true)
})
t.Run("BatchReplayWithDeleteRange", func(t *testing.T) {
db := New()
defer db.Close()
// Setup some initial data
for i := 1; i <= 10; i++ {
if err := db.Put([]byte(strconv.Itoa(i)), []byte("val-"+strconv.Itoa(i))); err != nil {
t.Fatal(err)
}
}
// Create batch with multiple operations including DeleteRange
batch1 := db.NewBatch()
batch1.Put([]byte("new-key-1"), []byte("new-val-1"))
batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7
batch1.DeleteRange([]byte("3"), []byte("7")) // Should delete keys 3-6 but not 7
batch1.Delete([]byte("8"))
batch1.Put([]byte("new-key-2"), []byte("new-val-2"))
// Create a second batch to replay into
batch2 := db.NewBatch()
if err := batch1.Replay(batch2); err != nil {
t.Fatal(err)
}
// Write the second batch
if err := batch2.Write(); err != nil {
t.Fatal(err)
}
// Verify results
// Original keys 3-6 should be deleted (inclusive of start, exclusive of end)
for i := 3; i <= 6; i++ {
@ -622,7 +645,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
t.Fatalf("key %d should be deleted", i)
}
}
// Key 7 should NOT be deleted (exclusive of end)
has, err := db.Has([]byte("7"))
if err != nil {
@ -631,7 +654,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if !has {
t.Fatalf("key 7 should NOT be deleted (exclusive of end)")
}
// Key 8 should be deleted
has, err = db.Has([]byte("8"))
if err != nil {
@ -640,7 +663,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
if has {
t.Fatalf("key 8 should be deleted")
}
// New keys should be added
for _, key := range []string{"new-key-1", "new-key-2"} {
has, err := db.Has([]byte(key))
@ -651,16 +674,16 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
t.Fatalf("key %s should exist", key)
}
}
// Create a third batch for direct replay to database
batch3 := db.NewBatch()
batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3
batch3.DeleteRange([]byte("1"), []byte("3")) // Should delete keys 1-2 but not 3
// Replay directly to the database
if err := batch3.Replay(db); err != nil {
t.Fatal(err)
}
// Verify keys 1-2 are now deleted
for i := 1; i <= 2; i++ {
has, err := db.Has([]byte(strconv.Itoa(i)))
@ -671,7 +694,7 @@ func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) {
t.Fatalf("key %d should be deleted after direct replay", i)
}
}
// Verify key 3 is NOT deleted (since it's exclusive of end)
has, err = db.Has([]byte("3"))
if err != nil {
@ -808,16 +831,16 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) {
for i := 0; i < count; i++ {
db.Put([]byte(strconv.Itoa(i)), nil)
}
b.ResetTimer()
b.ReportAllocs()
// Create batch and delete range
batch := db.NewBatch()
batch.DeleteRange([]byte("0"), []byte("999999999"))
batch.Write()
}
b.Run("BatchDeleteRange100", func(b *testing.B) {
benchBatchDeleteRange(b, 100)
})
@ -828,42 +851,42 @@ func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) {
benchBatchDeleteRange(b, 10000)
})
})
b.Run("BatchMixedOps", func(b *testing.B) {
benchBatchMixedOps := func(b *testing.B, count int) {
db := New()
defer db.Close()
// Prepare initial data
for i := 0; i < count; i++ {
db.Put([]byte(strconv.Itoa(i)), []byte("val"))
}
b.ResetTimer()
b.ReportAllocs()
// Create batch with mixed operations
batch := db.NewBatch()
// Add some new keys
for i := 0; i < count/10; i++ {
batch.Put([]byte(strconv.Itoa(count+i)), []byte("new-val"))
}
// Delete some individual keys
for i := 0; i < count/20; i++ {
batch.Delete([]byte(strconv.Itoa(i*2)))
batch.Delete([]byte(strconv.Itoa(i * 2)))
}
// Delete range of keys
rangeStart := count / 2
rangeEnd := count * 3 / 4
batch.DeleteRange([]byte(strconv.Itoa(rangeStart)), []byte(strconv.Itoa(rangeEnd)))
// Write the batch
batch.Write()
}
b.Run("BatchMixedOps100", func(b *testing.B) {
benchBatchMixedOps(b, 100)
})

48
ethdb/leveldb/leveldb.go
View file

@ -220,7 +220,7 @@ func (db *Database) DeleteRange(start, end []byte) error {
defer it.Release()
var count int
for it.Next() && bytes.Compare(end, it.Key()) > 0 {
for it.Next() && (end == nil || bytes.Compare(end, it.Key()) > 0) {
count++
if count > 10000 { // should not block for more than a second
if err := batch.Write(); err != nil {
@ -461,48 +461,19 @@ func (b *batch) Delete(key []byte) error {
return nil
}
// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
// DeleteRange removes all keys in the range [start, end) from the batch for
// later committing, inclusive on start, exclusive on end.
//
// Note that this is a fallback implementation as leveldb does not natively
// support range deletion in batches. It iterates through the database to find
// keys in the range and adds them to the batch for deletion.
func (b *batch) DeleteRange(start, end []byte) error {
// Special case: empty range
if len(start) == 0 && len(end) == 0 || bytes.Equal(start, end) {
return nil
}
// Special case: delete all keys less than end
if len(start) == 0 {
it := b.db.NewIterator(nil, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if bytes.Compare(key, end) < 0 {
b.b.Delete(key)
b.size += len(key)
}
}
return it.Error()
}
// Special case: delete all keys greater than or equal to start
if len(end) == 0 {
it := b.db.NewIterator(nil, nil)
defer it.Release()
for it.Next() {
key := it.Key()
if bytes.Compare(key, start) >= 0 {
b.b.Delete(key)
b.size += len(key)
}
}
return it.Error()
}
// Create an iterator to scan through the keys in the range
it := b.db.NewIterator(&util.Range{Start: start, Limit: end}, nil)
slice := &util.Range{
Start: start, // If nil, it represents the key before all keys
Limit: end, // If nil, it represents the key after all keys
}
it := b.db.NewIterator(slice, nil)
defer it.Release()
var count int
@ -516,7 +487,6 @@ func (b *batch) DeleteRange(start, end []byte) error {
b.b.Delete(key)
b.size += len(key)
}
if err := it.Error(); err != nil {
return err
}

116
ethdb/memorydb/memorydb.go
View file

@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
@ -124,46 +123,24 @@ func (db *Database) Delete(key []byte) error {
}
// DeleteRange deletes all of the keys (and values) in the range [start,end)
// (inclusive on start, exclusive on end).
// (inclusive on start, exclusive on end). If the start is nil, it represents
// the key before all keys; if the end is nil, it represents the key after
// all keys.
func (db *Database) DeleteRange(start, end []byte) error {
db.lock.Lock()
defer db.lock.Unlock()
if db.db == nil {
return errMemorydbClosed
}
startStr := string(start)
endStr := string(end)
if startStr == "" && endStr == "" {
return nil // Empty range, do nothing
}
if startStr == "" {
// Delete all keys less than end
for key := range db.db {
if key < endStr {
delete(db.db, key)
}
}
return nil
}
if endStr == "" || endStr > string([]byte{255, 255, 255, 255, 255}) {
// Delete all keys greater than or equal to start
for key := range db.db {
if key >= startStr {
delete(db.db, key)
}
}
return nil
}
// Normal case: delete keys in range [start, end)
for key := range db.db {
if key >= startStr && key < endStr {
delete(db.db, key)
if start != nil && key < string(start) {
continue
}
if end != nil && key >= string(end) {
continue
}
delete(db.db, key)
}
return nil
}
@ -249,11 +226,12 @@ func (db *Database) Len() int {
// keyvalue is a key-value tuple tagged with a deletion field to allow creating
// memory-database write batches.
type keyvalue struct {
key string
value []byte
delete bool
rangeFrom string
rangeTo string
key string
value []byte
delete bool
rangeFrom []byte
rangeTo []byte
}
// batch is a write-only memory batch that commits changes to its host
@ -281,8 +259,8 @@ func (b *batch) Delete(key []byte) error {
// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
func (b *batch) DeleteRange(start, end []byte) error {
b.writes = append(b.writes, keyvalue{
rangeFrom: string(start),
rangeTo: string(end),
rangeFrom: start,
rangeTo: end,
delete: true,
})
b.size += len(start) + len(end)
@ -302,46 +280,26 @@ func (b *batch) Write() error {
if b.db.db == nil {
return errMemorydbClosed
}
for _, keyvalue := range b.writes {
if keyvalue.delete {
if keyvalue.key != "" {
for _, entry := range b.writes {
if entry.delete {
if entry.key != "" {
// Single key deletion
delete(b.db.db, keyvalue.key)
} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
delete(b.db.db, entry.key)
} else {
// Range deletion (inclusive of start, exclusive of end)
// Handle special cases for empty ranges
if keyvalue.rangeFrom == keyvalue.rangeTo {
// Empty range, do nothing
continue
}
// Check if both are numeric strings for consistent comparison
startNum, startErr := strconv.Atoi(keyvalue.rangeFrom)
endNum, endErr := strconv.Atoi(keyvalue.rangeTo)
// If both are valid integers, use numeric comparison
if startErr == nil && endErr == nil {
for key := range b.db.db {
if keyNum, err := strconv.Atoi(key); err == nil {
if keyNum >= startNum && (keyvalue.rangeTo == "" || keyNum < endNum) {
delete(b.db.db, key)
}
}
for key := range b.db.db {
if entry.rangeFrom != nil && key < string(entry.rangeFrom) {
continue
}
} else {
// Use string comparison
for key := range b.db.db {
// Handle open ranges
if (keyvalue.rangeFrom == "" || key >= keyvalue.rangeFrom) &&
(keyvalue.rangeTo == "" || key < keyvalue.rangeTo) {
delete(b.db.db, key)
}
if entry.rangeTo != nil && key >= string(entry.rangeTo) {
continue
}
delete(b.db.db, key)
}
}
continue
}
b.db.db[keyvalue.key] = keyvalue.value
b.db.db[entry.key] = entry.value
}
return nil
}
@ -354,17 +312,17 @@ func (b *batch) Reset() {
// Replay replays the batch contents.
func (b *batch) Replay(w ethdb.KeyValueWriter) error {
for _, keyvalue := range b.writes {
if keyvalue.delete {
if keyvalue.key != "" {
for _, entry := range b.writes {
if entry.delete {
if entry.key != "" {
// Single key deletion
if err := w.Delete([]byte(keyvalue.key)); err != nil {
if err := w.Delete([]byte(entry.key)); err != nil {
return err
}
} else if keyvalue.rangeFrom != "" || keyvalue.rangeTo != "" {
} else {
// Range deletion
if rangeDeleter, ok := w.(ethdb.KeyValueRangeDeleter); ok {
if err := rangeDeleter.DeleteRange([]byte(keyvalue.rangeFrom), []byte(keyvalue.rangeTo)); err != nil {
if err := rangeDeleter.DeleteRange(entry.rangeFrom, entry.rangeTo); err != nil {
return err
}
} else {
@ -373,7 +331,7 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error {
}
continue
}
if err := w.Put([]byte(keyvalue.key), keyvalue.value); err != nil {
if err := w.Put([]byte(entry.key), entry.value); err != nil {
return err
}
}

16
ethdb/pebble/pebble.go
View file

@ -403,9 +403,16 @@ func (d *Database) Delete(key []byte) error {
func (d *Database) DeleteRange(start, end []byte) error {
d.quitLock.RLock()
defer d.quitLock.RUnlock()
if d.closed {
return pebble.ErrClosed
}
// There is no special flag to represent the end of key range
// in pebble(nil in leveldb). Use an ugly hack to construct a
// large key to represent it.
if end == nil {
end = bytes.Repeat([]byte{0xff}, 32)
}
return d.db.DeleteRange(start, end, d.writeOptions)
}
@ -619,8 +626,15 @@ func (b *batch) Delete(key []byte) error {
return nil
}
// DeleteRange removes all keys in the range [start, end) from the batch for later committing.
// DeleteRange removes all keys in the range [start, end) from the batch for
// later committing, inclusive on start, exclusive on end.
func (b *batch) DeleteRange(start, end []byte) error {
// There is no special flag to represent the end of key range
// in pebble(nil in leveldb). Use an ugly hack to construct a
// large key to represent it.
if end == nil {
end = bytes.Repeat([]byte{0xff}, 32)
}
if err := b.b.DeleteRange(start, end, nil); err != nil {
return err
}