swamr/shed: only one Index Iterate function

swarm/shed/example_store_test.go

@@ -244,7 +244,7 @@ func (s *Store) CollectGarbage() (err error) {
 		// New batch for a new cg round.
 		trash := new(leveldb.Batch)
 		// Iterate through all index items and break when needed.
-		err = s.gcIndex.IterateAll(func(item shed.Item) (stop bool, err error) {
+		err = s.gcIndex.Iterate(func(item shed.Item) (stop bool, err error) {
 			// Remove the chunk.
 			err = s.retrievalIndex.DeleteInBatch(trash, item)
 			if err != nil {
@@ -265,7 +265,7 @@ func (s *Store) CollectGarbage() (err error) {
 				return true, nil
 			}
 			return false, nil
-		})
+		}, nil)
 		if err != nil {
 			return err
 		}

swarm/shed/index.go

@@ -202,118 +202,51 @@ func (f Index) DeleteInBatch(batch *leveldb.Batch, keyFields Item) (err error) {
 // propagated to the called iterator method on Index.
 type IndexIterFunc func(item Item) (stop bool, err error)
 
-// IterateAll iterates over all keys of the Index.
+// IterateOptions defines optional parameters for Iterate function.
-func (f Index) IterateAll(fn IndexIterFunc) (err error) {
+type IterateOptions struct {
+	// StartFrom is the Item to start the iteration from.
+	StartFrom *Item
+	// If SkipStartFromItem is true, StartFrom item will not
+	// be iterated on.
+	SkipStartFromItem bool
+	// Iterate over items which keys have a common prefix.
+	Prefix []byte
+}
+
+// Iterate function iterates over keys of the Index.
+// If IterateOptions is nil, the iterations is over all keys.
+func (f Index) Iterate(fn IndexIterFunc, options *IterateOptions) (err error) {
+	if options == nil {
+		options = new(IterateOptions)
+	}
+	// construct a prefix with Index prefix and optional common key prefix
+	prefix := append(f.prefix, options.Prefix...)
+	// start from the prefix
+	startKey := prefix
+	if options.StartFrom != nil {
+		// start from the provided StartFrom Item key value
+		startKey, err = f.encodeKeyFunc(*options.StartFrom)
+		if err != nil {
+			return err
+		}
+	}
 	it := f.db.NewIterator()
 	defer it.Release()
 
-	for ok := it.Seek(f.prefix); ok; ok = it.Next() {
+	// move the cursor to the start key
-		key := it.Key()
+	ok := it.Seek(startKey)
-		if key[0] != f.prefix[0] {
+	if !ok {
-			break
+		// stop iterator if seek has failed
-		}
-		keyItem, err := f.decodeKeyFunc(key)
-		if err != nil {
-			return err
-		}
-		valueItem, err := f.decodeValueFunc(keyItem, it.Value())
-		if err != nil {
-			return err
-		}
-		stop, err := fn(keyItem.Merge(valueItem))
-		if err != nil {
-			return err
-		}
-		if stop {
-			break
-		}
-	}
 		return it.Error()
 	}
-
+	if options.SkipStartFromItem && bytes.Equal(startKey, it.Key()) {
-// IterateFrom iterates over Index keys starting from the key
+		// skip the start from Item if it is the first key
-// encoded from the provided Item.
+		// and it is explicitly configured to skip it
-func (f Index) IterateFrom(start Item, fn IndexIterFunc) (err error) {
+		ok = it.Next()
-	startKey, err := f.encodeKeyFunc(start)
-	if err != nil {
-		return err
 	}
-	it := f.db.NewIterator()
+	for ; ok; ok = it.Next() {
-	defer it.Release()
-
-	for ok := it.Seek(startKey); ok; ok = it.Next() {
 		key := it.Key()
-		if key[0] != f.prefix[0] {
+		if !bytes.HasPrefix(key, prefix) {
-			break
-		}
-		keyItem, err := f.decodeKeyFunc(key)
-		if err != nil {
-			return err
-		}
-		valueItem, err := f.decodeValueFunc(keyItem, it.Value())
-		if err != nil {
-			return err
-		}
-		stop, err := fn(keyItem.Merge(valueItem))
-		if err != nil {
-			return err
-		}
-		if stop {
-			break
-		}
-	}
-	return it.Error()
-}
-
-// IterateWithPrefix iterates over all keys of the Index that have
-// a common prefix.
-func (f Index) IterateWithPrefix(prefix []byte, fn IndexIterFunc) (err error) {
-	it := f.db.NewIterator()
-	defer it.Release()
-
-	// construct complete prefix with index prefix
-	p := append(f.prefix, prefix...)
-
-	for ok := it.Seek(p); ok; ok = it.Next() {
-		key := it.Key()
-		if !bytes.HasPrefix(key, p) {
-			break
-		}
-		keyItem, err := f.decodeKeyFunc(key)
-		if err != nil {
-			return err
-		}
-		valueItem, err := f.decodeValueFunc(keyItem, it.Value())
-		if err != nil {
-			return err
-		}
-		stop, err := fn(keyItem.Merge(valueItem))
-		if err != nil {
-			return err
-		}
-		if stop {
-			break
-		}
-	}
-	return it.Error()
-}
-
-// IterateWithPrefixFrom iterates over Index keys that have a common prefix,
-// starting from the key encoded from the provided start Item.
-func (f Index) IterateWithPrefixFrom(prefix []byte, start Item, fn IndexIterFunc) (err error) {
-	startKey, err := f.encodeKeyFunc(start)
-	if err != nil {
-		return err
-	}
-	it := f.db.NewIterator()
-	defer it.Release()
-
-	// construct complete prefix with index prefix
-	p := append(f.prefix, prefix...)
-
-	for ok := it.Seek(startKey); ok; ok = it.Next() {
-		key := it.Key()
-		if !bytes.HasPrefix(key, p) {
 			break
 		}
 		keyItem, err := f.decodeKeyFunc(key)

swarm/shed/index_test.go

@@ -250,9 +250,9 @@ func TestIndex(t *testing.T) {
 	})
 }
 
-// TestIndex_iterate validates index IterateAll and IterateFrom
+// TestIndex_Iterate validates index Iterate
 // functions for correctness.
-func TestIndex_iterate(t *testing.T) {
+func TestIndex_Iterate(t *testing.T) {
 	db, cleanupFunc := newTestDB(t)
 	defer cleanupFunc()
 
@@ -307,7 +307,7 @@ func TestIndex_iterate(t *testing.T) {
 
 	t.Run("all", func(t *testing.T) {
 		var i int
-		err := index.IterateAll(func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -315,16 +315,35 @@ func TestIndex_iterate(t *testing.T) {
 			checkItem(t, item, want)
 			i++
 			return false, nil
+		}, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+
+	t.Run("start from", func(t *testing.T) {
+		startIndex := 2
+		i := startIndex
+		err := index.Iterate(func(item Item) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkItem(t, item, want)
+			i++
+			return false, nil
+		}, &IterateOptions{
+			StartFrom: &items[startIndex],
 		})
 		if err != nil {
 			t.Fatal(err)
 		}
 	})
 
-	t.Run("from", func(t *testing.T) {
+	t.Run("skip start from", func(t *testing.T) {
 		startIndex := 2
-		i := startIndex
+		i := startIndex + 1
-		err := index.IterateFrom(items[startIndex], func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -332,6 +351,9 @@ func TestIndex_iterate(t *testing.T) {
 			checkItem(t, item, want)
 			i++
 			return false, nil
+		}, &IterateOptions{
+			StartFrom:         &items[startIndex],
+			SkipStartFromItem: true,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -342,7 +364,7 @@ func TestIndex_iterate(t *testing.T) {
 		var i int
 		stopIndex := 3
 		var count int
-		err := index.IterateAll(func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -354,7 +376,7 @@ func TestIndex_iterate(t *testing.T) {
 			}
 			i++
 			return false, nil
-		})
+		}, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -380,7 +402,7 @@ func TestIndex_iterate(t *testing.T) {
 		}
 
 		var i int
-		err = index.IterateAll(func(item Item) (stop bool, err error) {
+		err = index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -388,29 +410,29 @@ func TestIndex_iterate(t *testing.T) {
 			checkItem(t, item, want)
 			i++
 			return false, nil
-		})
+		}, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
 
 		i = 0
-		err = secondIndex.IterateAll(func(item Item) (stop bool, err error) {
+		err = secondIndex.Iterate(func(item Item) (stop bool, err error) {
 			if i > 1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
 			checkItem(t, item, secondItem)
 			i++
 			return false, nil
-		})
+		}, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
 	})
 }
 
-// TestIndex_iterateWithPrefix validates index IterateWithPrefix
+// TestIndex_Iterate_withPrefix validates index Iterate
-// and IterateWithPrefixFrom functions for correctness.
+// function for correctness.
-func TestIndex_iterateWithPrefix(t *testing.T) {
+func TestIndex_Iterate_withPrefix(t *testing.T) {
 	db, cleanupFunc := newTestDB(t)
 	defer cleanupFunc()
 
@@ -455,7 +477,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 
 	t.Run("with prefix", func(t *testing.T) {
 		var i int
-		err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -463,6 +485,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 			checkItem(t, item, want)
 			i++
 			return false, nil
+		}, &IterateOptions{
+			Prefix: prefix,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -472,11 +496,11 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 		}
 	})
 
-	t.Run("with prefix from", func(t *testing.T) {
+	t.Run("with prefix and start from", func(t *testing.T) {
 		startIndex := 2
 		var count int
 		i := startIndex
-		err := index.IterateWithPrefixFrom(prefix, items[startIndex], func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -485,6 +509,9 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 			i++
 			count++
 			return false, nil
+		}, &IterateOptions{
+			StartFrom: &items[startIndex],
+			Prefix:    prefix,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -495,11 +522,38 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 		}
 	})
 
+	t.Run("with prefix and skip start from", func(t *testing.T) {
+		startIndex := 2
+		var count int
+		i := startIndex + 1
+		err := index.Iterate(func(item Item) (stop bool, err error) {
+			if i > len(items)-1 {
+				return true, fmt.Errorf("got unexpected index item: %#v", item)
+			}
+			want := items[i]
+			checkItem(t, item, want)
+			i++
+			count++
+			return false, nil
+		}, &IterateOptions{
+			StartFrom:         &items[startIndex],
+			SkipStartFromItem: true,
+			Prefix:            prefix,
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+		wantCount := len(items) - startIndex - 1
+		if count != wantCount {
+			t.Errorf("got %v items, want %v", count, wantCount)
+		}
+	})
+
 	t.Run("stop", func(t *testing.T) {
 		var i int
 		stopIndex := 3
 		var count int
-		err := index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
+		err := index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -511,6 +565,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 			}
 			i++
 			return false, nil
+		}, &IterateOptions{
+			Prefix: prefix,
 		})
 		if err != nil {
 			t.Fatal(err)
@@ -537,7 +593,7 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 		}
 
 		var i int
-		err = index.IterateWithPrefix(prefix, func(item Item) (stop bool, err error) {
+		err = index.Iterate(func(item Item) (stop bool, err error) {
 			if i > len(items)-1 {
 				return true, fmt.Errorf("got unexpected index item: %#v", item)
 			}
@@ -545,6 +601,8 @@ func TestIndex_iterateWithPrefix(t *testing.T) {
 			checkItem(t, item, want)
 			i++
 			return false, nil
+		}, &IterateOptions{
+			Prefix: prefix,
 		})
 		if err != nil {
 			t.Fatal(err)