forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 18:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

swamr/shed: only one Index Iterate function

Browse source
This commit is contained in:
Janos Guljas 2018-12-19 22:32:30 +01:00
parent da9bab4a7d
commit d54d7ae0f8
3 changed files with 116 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
swarm/shed/example_store_test.go
View file

@ -244,7 +244,7 @@ func (s *Store) CollectGarbage() (err error) {
// New batch for a new cg round.
trash := new(leveldb.Batch)
// Iterate through all index items and break when needed.
err = s.gcIndex.IterateAll(func(item shed.Item) (stop bool, err error) {
err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
// Remove the chunk.
err = s.retrievalIndex.DeleteInBatch(trash, item)
if err != nil {
@ -265,7 +265,7 @@ func (s *Store) CollectGarbage() (err error) {
return true, nil
}
return false, nil
})
}, nil)
if err != nil {
return err
}

139
swarm/shed/index.go
View file

@ -202,118 +202,51 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields Item) (err error) {
// propagated to the called iterator method on Index.
type IndexIterFunc func(item Item) (stop bool, err error)
// IterateAll iterates over all keys of the Index.
func (f Index) IterateAll(fn IndexIterFunc) (err error) {
it := f.db.NewIterator()
defer it.Release()
for ok := it.Seek(f.prefix); ok; ok = it.Next() {
key := it.Key()
if key[0] != f.prefix[0] {
break
}
keyItem, err := f.decodeKeyFunc(key)
if err != nil {
return err
}
valueItem, err := f.decodeValueFunc(keyItem, it.Value())
if err != nil {
return err
}
stop, err := fn(keyItem.Merge(valueItem))
if err != nil {
return err
}
if stop {
break
}
}
return it.Error()
// IterateOptions defines optional parameters for Iterate function.
type IterateOptions struct {
// StartFrom is the Item to start the iteration from.
StartFrom *Item
// If SkipStartFromItem is true, StartFrom item will not
// be iterated on.
SkipStartFromItem bool
// Iterate over items which keys have a common prefix.
Prefix []byte
}
// IterateFrom iterates over Index keys starting from the key
// encoded from the provided Item.
func (f Index) IterateFrom(start Item, fn IndexIterFunc) (err error) {
startKey, err := f.encodeKeyFunc(start)
if err != nil {
return err
// Iterate function iterates over keys of the Index.
// If IterateOptions is nil, the iterations is over all keys.
func (f Index) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) {
if options == nil {
options = new(IterateOptions)
}
// construct a prefix with Index prefix and optional common key prefix
prefix := append(f.prefix, options.Prefix...)
// start from the prefix
startKey := prefix
if options.StartFrom != nil {
// start from the provided StartFrom Item key value
startKey, err = f.encodeKeyFunc(*options.StartFrom)
if err != nil {
return err
}
}
it := f.db.NewIterator()
defer it.Release()
for ok := it.Seek(startKey); ok; ok = it.Next() {
key := it.Key()
if key[0] != f.prefix[0] {
break
}
keyItem, err := f.decodeKeyFunc(key)
if err != nil {
return err
}
valueItem, err := f.decodeValueFunc(keyItem, it.Value())
if err != nil {
return err
}
stop, err := fn(keyItem.Merge(valueItem))
if err != nil {
return err
}
if stop {
break
}
// move the cursor to the start key
ok := it.Seek(startKey)
if !ok {
// stop iterator if seek has failed
return it.Error()
}
return it.Error()
}
// IterateWithPrefix iterates over all keys of the Index that have
// a common prefix.
func (f Index) IterateWithPrefix(prefix []byte, fn IndexIterFunc) (err error) {
it := f.db.NewIterator()
defer it.Release()
// construct complete prefix with index prefix
p := append(f.prefix, prefix...)
for ok := it.Seek(p); ok; ok = it.Next() {
key := it.Key()
if !bytes.HasPrefix(key, p) {
break
}
keyItem, err := f.decodeKeyFunc(key)
if err != nil {
return err
}
valueItem, err := f.decodeValueFunc(keyItem, it.Value())
if err != nil {
return err
}
stop, err := fn(keyItem.Merge(valueItem))
if err != nil {
return err
}
if stop {
break
}
if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) {
// skip the start from Item if it is the first key
// and it is explicitly configured to skip it
ok = it.Next()
}
return it.Error()
}
// IterateWithPrefixFrom iterates over Index keys that have a common prefix,
// starting from the key encoded from the provided start Item.
func (f Index) IterateWithPrefixFrom(prefix []byte, start Item, fn IndexIterFunc) (err error) {
startKey, err := f.encodeKeyFunc(start)
if err != nil {
return err
}
it := f.db.NewIterator()
defer it.Release()
// construct complete prefix with index prefix
p := append(f.prefix, prefix...)
for ok := it.Seek(startKey); ok; ok = it.Next() {
for ; ok; ok = it.Next() {
key := it.Key()
if !bytes.HasPrefix(key, p) {
if !bytes.HasPrefix(key, prefix) {
break
}
keyItem, err := f.decodeKeyFunc(key)

98
swarm/shed/index_test.go
View file

@ -250,9 +250,9 @@ func TestIndex(t *testing.T) {
})
}
// TestIndex_iterate validates index IterateAll and IterateFrom
// TestIndex_Iterate validates index Iterate
// functions for correctness.
func TestIndex_iterate(t *testing.T) {
func TestIndex_Iterate(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -307,7 +307,7 @@ func TestIndex_iterate(t *testing.T) {
t.Run("all", func(t *testing.T) {
var i int
err := index.IterateAll(func(item Item) (stop bool, err error) {
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -315,16 +315,35 @@ func TestIndex_iterate(t *testing.T) {
checkItem(t, item, want)
i++
return false, nil
}, nil)
if err != nil {
t.Fatal(err)
}
})
t.Run("start from", func(t *testing.T) {
startIndex := 2
i := startIndex
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
want := items[i]
checkItem(t, item, want)
i++
return false, nil
}, &IterateOptions{
StartFrom: &items[startIndex],
})
if err != nil {
t.Fatal(err)
}
})
t.Run("from", func(t *testing.T) {
t.Run("skip start from", func(t *testing.T) {
startIndex := 2
i := startIndex
err := index.IterateFrom(items[startIndex], func(item Item) (stop bool, err error) {
i := startIndex + 1
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -332,6 +351,9 @@ func TestIndex_iterate(t *testing.T) {
checkItem(t, item, want)
i++
return false, nil
}, &IterateOptions{
StartFrom: &items[startIndex],
SkipStartFromItem: true,
})
if err != nil {
t.Fatal(err)
@ -342,7 +364,7 @@ func TestIndex_iterate(t *testing.T) {
var i int
stopIndex := 3
var count int
err := index.IterateAll(func(item Item) (stop bool, err error) {
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -354,7 +376,7 @@ func TestIndex_iterate(t *testing.T) {
}
i++
return false, nil
})
}, nil)
if err != nil {
t.Fatal(err)
}
@ -380,7 +402,7 @@ func TestIndex_iterate(t *testing.T) {
}
var i int
err = index.IterateAll(func(item Item) (stop bool, err error) {
err = index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -388,29 +410,29 @@ func TestIndex_iterate(t *testing.T) {
checkItem(t, item, want)
i++
return false, nil
})
}, nil)
if err != nil {
t.Fatal(err)
}
i = 0
err = secondIndex.IterateAll(func(item Item) (stop bool, err error) {
err = secondIndex.Iterate(func(item Item) (stop bool, err error) {
if i > 1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
checkItem(t, item, secondItem)
i++
return false, nil
})
}, nil)
if err != nil {
t.Fatal(err)
}
})
}
// TestIndex_iterateWithPrefix validates index IterateWithPrefix
// and IterateWithPrefixFrom functions for correctness.
func TestIndex_iterateWithPrefix(t *testing.T) {
// TestIndex_Iterate_withPrefix validates index Iterate
// function for correctness.
func TestIndex_Iterate_withPrefix(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -455,7 +477,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
t.Run("with prefix", func(t *testing.T) {
var i int
err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -463,6 +485,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
checkItem(t, item, want)
i++
return false, nil
}, &IterateOptions{
Prefix: prefix,
})
if err != nil {
t.Fatal(err)
@ -472,11 +496,11 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
}
})
t.Run("with prefix from", func(t *testing.T) {
t.Run("with prefix and start from", func(t *testing.T) {
startIndex := 2
var count int
i := startIndex
err := index.IterateWithPrefixFrom(prefix, items[startIndex], func(item Item) (stop bool, err error) {
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -485,6 +509,9 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
i++
count++
return false, nil
}, &IterateOptions{
StartFrom: &items[startIndex],
Prefix: prefix,
})
if err != nil {
t.Fatal(err)
@ -495,11 +522,38 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
}
})
t.Run("with prefix and skip start from", func(t *testing.T) {
startIndex := 2
var count int
i := startIndex + 1
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
want := items[i]
checkItem(t, item, want)
i++
count++
return false, nil
}, &IterateOptions{
StartFrom: &items[startIndex],
SkipStartFromItem: true,
Prefix: prefix,
})
if err != nil {
t.Fatal(err)
}
wantCount := len(items) - startIndex - 1
if count != wantCount {
t.Errorf("got %v items, want %v", count, wantCount)
}
})
t.Run("stop", func(t *testing.T) {
var i int
stopIndex := 3
var count int
err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
err := index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -511,6 +565,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
}
i++
return false, nil
}, &IterateOptions{
Prefix: prefix,
})
if err != nil {
t.Fatal(err)
@ -537,7 +593,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
}
var i int
err = index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
err = index.Iterate(func(item Item) (stop bool, err error) {
if i > len(items)-1 {
return true, fmt.Errorf("got unexpected index item: %#v", item)
}
@ -545,6 +601,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
checkItem(t, item, want)
i++
return false, nil
}, &IterateOptions{
Prefix: prefix,
})
if err != nil {
t.Fatal(err)