From d54d7ae0f8cba174d66f93ba3ec61e76f1afd819 Mon Sep 17 00:00:00 2001 From: Janos Guljas Date: Wed, 19 Dec 2018 22:32:30 +0100 Subject: [PATCH] swamr/shed: only one Index Iterate function --- swarm/shed/example_store_test.go | 4 +- swarm/shed/index.go | 139 ++++++++----------------------- swarm/shed/index_test.go | 98 +++++++++++++++++----- 3 files changed, 116 insertions(+), 125 deletions(-) diff --git a/swarm/shed/example_store_test.go b/swarm/shed/example_store_test.go index aeff4c1142..9a83855e70 100644 --- a/swarm/shed/example_store_test.go +++ b/swarm/shed/example_store_test.go @@ -244,7 +244,7 @@ func (s *Store) CollectGarbage() (err error) { // New batch for a new cg round. trash := new(leveldb.Batch) // Iterate through all index items and break when needed. - err = s.gcIndex.IterateAll(func(item shed.Item) (stop bool, err error) { + err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) { // Remove the chunk. err = s.retrievalIndex.DeleteInBatch(trash, item) if err != nil { @@ -265,7 +265,7 @@ func (s *Store) CollectGarbage() (err error) { return true, nil } return false, nil - }) + }, nil) if err != nil { return err } diff --git a/swarm/shed/index.go b/swarm/shed/index.go index c592dc570e..a1646e9f26 100644 --- a/swarm/shed/index.go +++ b/swarm/shed/index.go @@ -202,118 +202,51 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields Item) (err error) { // propagated to the called iterator method on Index. type IndexIterFunc func(item Item) (stop bool, err error) -// IterateAll iterates over all keys of the Index. -func (f Index) IterateAll(fn IndexIterFunc) (err error) { - it := f.db.NewIterator() - defer it.Release() - - for ok := it.Seek(f.prefix); ok; ok = it.Next() { - key := it.Key() - if key[0] != f.prefix[0] { - break - } - keyItem, err := f.decodeKeyFunc(key) - if err != nil { - return err - } - valueItem, err := f.decodeValueFunc(keyItem, it.Value()) - if err != nil { - return err - } - stop, err := fn(keyItem.Merge(valueItem)) - if err != nil { - return err - } - if stop { - break - } - } - return it.Error() +// IterateOptions defines optional parameters for Iterate function. +type IterateOptions struct { + // StartFrom is the Item to start the iteration from. + StartFrom *Item + // If SkipStartFromItem is true, StartFrom item will not + // be iterated on. + SkipStartFromItem bool + // Iterate over items which keys have a common prefix. + Prefix []byte } -// IterateFrom iterates over Index keys starting from the key -// encoded from the provided Item. -func (f Index) IterateFrom(start Item, fn IndexIterFunc) (err error) { - startKey, err := f.encodeKeyFunc(start) - if err != nil { - return err +// Iterate function iterates over keys of the Index. +// If IterateOptions is nil, the iterations is over all keys. +func (f Index) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) { + if options == nil { + options = new(IterateOptions) + } + // construct a prefix with Index prefix and optional common key prefix + prefix := append(f.prefix, options.Prefix...) + // start from the prefix + startKey := prefix + if options.StartFrom != nil { + // start from the provided StartFrom Item key value + startKey, err = f.encodeKeyFunc(*options.StartFrom) + if err != nil { + return err + } } it := f.db.NewIterator() defer it.Release() - for ok := it.Seek(startKey); ok; ok = it.Next() { - key := it.Key() - if key[0] != f.prefix[0] { - break - } - keyItem, err := f.decodeKeyFunc(key) - if err != nil { - return err - } - valueItem, err := f.decodeValueFunc(keyItem, it.Value()) - if err != nil { - return err - } - stop, err := fn(keyItem.Merge(valueItem)) - if err != nil { - return err - } - if stop { - break - } + // move the cursor to the start key + ok := it.Seek(startKey) + if !ok { + // stop iterator if seek has failed + return it.Error() } - return it.Error() -} - -// IterateWithPrefix iterates over all keys of the Index that have -// a common prefix. -func (f Index) IterateWithPrefix(prefix []byte, fn IndexIterFunc) (err error) { - it := f.db.NewIterator() - defer it.Release() - - // construct complete prefix with index prefix - p := append(f.prefix, prefix...) - - for ok := it.Seek(p); ok; ok = it.Next() { - key := it.Key() - if !bytes.HasPrefix(key, p) { - break - } - keyItem, err := f.decodeKeyFunc(key) - if err != nil { - return err - } - valueItem, err := f.decodeValueFunc(keyItem, it.Value()) - if err != nil { - return err - } - stop, err := fn(keyItem.Merge(valueItem)) - if err != nil { - return err - } - if stop { - break - } + if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) { + // skip the start from Item if it is the first key + // and it is explicitly configured to skip it + ok = it.Next() } - return it.Error() -} - -// IterateWithPrefixFrom iterates over Index keys that have a common prefix, -// starting from the key encoded from the provided start Item. -func (f Index) IterateWithPrefixFrom(prefix []byte, start Item, fn IndexIterFunc) (err error) { - startKey, err := f.encodeKeyFunc(start) - if err != nil { - return err - } - it := f.db.NewIterator() - defer it.Release() - - // construct complete prefix with index prefix - p := append(f.prefix, prefix...) - - for ok := it.Seek(startKey); ok; ok = it.Next() { + for ; ok; ok = it.Next() { key := it.Key() - if !bytes.HasPrefix(key, p) { + if !bytes.HasPrefix(key, prefix) { break } keyItem, err := f.decodeKeyFunc(key) diff --git a/swarm/shed/index_test.go b/swarm/shed/index_test.go index 87f4e4e999..97d7c91f43 100644 --- a/swarm/shed/index_test.go +++ b/swarm/shed/index_test.go @@ -250,9 +250,9 @@ func TestIndex(t *testing.T) { }) } -// TestIndex_iterate validates index IterateAll and IterateFrom +// TestIndex_Iterate validates index Iterate // functions for correctness. -func TestIndex_iterate(t *testing.T) { +func TestIndex_Iterate(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -307,7 +307,7 @@ func TestIndex_iterate(t *testing.T) { t.Run("all", func(t *testing.T) { var i int - err := index.IterateAll(func(item Item) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -315,16 +315,35 @@ func TestIndex_iterate(t *testing.T) { checkItem(t, item, want) i++ return false, nil + }, nil) + if err != nil { + t.Fatal(err) + } + }) + + t.Run("start from", func(t *testing.T) { + startIndex := 2 + i := startIndex + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], }) if err != nil { t.Fatal(err) } }) - t.Run("from", func(t *testing.T) { + t.Run("skip start from", func(t *testing.T) { startIndex := 2 - i := startIndex - err := index.IterateFrom(items[startIndex], func(item Item) (stop bool, err error) { + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -332,6 +351,9 @@ func TestIndex_iterate(t *testing.T) { checkItem(t, item, want) i++ return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, }) if err != nil { t.Fatal(err) @@ -342,7 +364,7 @@ func TestIndex_iterate(t *testing.T) { var i int stopIndex := 3 var count int - err := index.IterateAll(func(item Item) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -354,7 +376,7 @@ func TestIndex_iterate(t *testing.T) { } i++ return false, nil - }) + }, nil) if err != nil { t.Fatal(err) } @@ -380,7 +402,7 @@ func TestIndex_iterate(t *testing.T) { } var i int - err = index.IterateAll(func(item Item) (stop bool, err error) { + err = index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -388,29 +410,29 @@ func TestIndex_iterate(t *testing.T) { checkItem(t, item, want) i++ return false, nil - }) + }, nil) if err != nil { t.Fatal(err) } i = 0 - err = secondIndex.IterateAll(func(item Item) (stop bool, err error) { + err = secondIndex.Iterate(func(item Item) (stop bool, err error) { if i > 1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } checkItem(t, item, secondItem) i++ return false, nil - }) + }, nil) if err != nil { t.Fatal(err) } }) } -// TestIndex_iterateWithPrefix validates index IterateWithPrefix -// and IterateWithPrefixFrom functions for correctness. -func TestIndex_iterateWithPrefix(t *testing.T) { +// TestIndex_Iterate_withPrefix validates index Iterate +// function for correctness. +func TestIndex_Iterate_withPrefix(t *testing.T) { db, cleanupFunc := newTestDB(t) defer cleanupFunc() @@ -455,7 +477,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) { t.Run("with prefix", func(t *testing.T) { var i int - err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -463,6 +485,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) { checkItem(t, item, want) i++ return false, nil + }, &IterateOptions{ + Prefix: prefix, }) if err != nil { t.Fatal(err) @@ -472,11 +496,11 @@ func TestIndex_iterateWithPrefix(t *testing.T) { } }) - t.Run("with prefix from", func(t *testing.T) { + t.Run("with prefix and start from", func(t *testing.T) { startIndex := 2 var count int i := startIndex - err := index.IterateWithPrefixFrom(prefix, items[startIndex], func(item Item) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -485,6 +509,9 @@ func TestIndex_iterateWithPrefix(t *testing.T) { i++ count++ return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + Prefix: prefix, }) if err != nil { t.Fatal(err) @@ -495,11 +522,38 @@ func TestIndex_iterateWithPrefix(t *testing.T) { } }) + t.Run("with prefix and skip start from", func(t *testing.T) { + startIndex := 2 + var count int + i := startIndex + 1 + err := index.Iterate(func(item Item) (stop bool, err error) { + if i > len(items)-1 { + return true, fmt.Errorf("got unexpected index item: %#v", item) + } + want := items[i] + checkItem(t, item, want) + i++ + count++ + return false, nil + }, &IterateOptions{ + StartFrom: &items[startIndex], + SkipStartFromItem: true, + Prefix: prefix, + }) + if err != nil { + t.Fatal(err) + } + wantCount := len(items) - startIndex - 1 + if count != wantCount { + t.Errorf("got %v items, want %v", count, wantCount) + } + }) + t.Run("stop", func(t *testing.T) { var i int stopIndex := 3 var count int - err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) { + err := index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -511,6 +565,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) { } i++ return false, nil + }, &IterateOptions{ + Prefix: prefix, }) if err != nil { t.Fatal(err) @@ -537,7 +593,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) { } var i int - err = index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) { + err = index.Iterate(func(item Item) (stop bool, err error) { if i > len(items)-1 { return true, fmt.Errorf("got unexpected index item: %#v", item) } @@ -545,6 +601,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) { checkItem(t, item, want) i++ return false, nil + }, &IterateOptions{ + Prefix: prefix, }) if err != nil { t.Fatal(err)