swarm/storage: Perform CleanGCIndex iteration in chunks

swarm/storage/ldbstore.go

@@ -602,14 +602,29 @@ func (s *LDBStore) CleanGCIndex() error {
 		return err
 	}
 
-	it.Seek([]byte{keyIndex})
+	it.Release()
-	var idx dpaDBIndex
+
+	//var idx dpaDBIndex
 	var poPtrs [256]uint64
-	for it.Valid() {
+	var doneIterating bool
-		rowType, chunkHash := parseGCIdxKey(it.Key())
+	for !doneIterating {
-		if rowType != keyIndex {
+		var idxs []dpaDBIndex
+		var chunkHashes [][]byte
+		var pos []uint8
+		it := s.db.NewIterator()
+		it.Seek([]byte{keyIndex})
+		for i := 0; i < 4096; i++ {
+			if !it.Valid() {
+				doneIterating = true
 				break
 			}
+			rowType, chunkHash := parseGCIdxKey(it.Key())
+			if rowType != keyIndex {
+				doneIterating = true
+				break
+			}
+			//err := decodeIndex(it.Value(), &idx)
+			var idx dpaDBIndex
 			err := decodeIndex(it.Value(), &idx)
 			if err != nil {
 				return fmt.Errorf("corrupt index: %v", err)
@@ -622,24 +637,41 @@ func (s *LDBStore) CleanGCIndex() error {
 			if err != nil {
 				log.Warn("deleting inconsistent index (missing data)", "key", chunkHash)
 				batch.Delete(it.Key())
+				//				if err := s.db.Delete(it.Key()); err != nil {
+				//					return err
+				//				}
 			} else {
-			gcIdxKey := getGCIdxKey(&idx)
+				idxs = append(idxs, idx)
-			gcIdxData := getGCIdxValue(&idx, po, chunkHash)
+				chunkHashes = append(chunkHashes, chunkHash)
-			batch.Put(gcIdxKey, gcIdxData)
+				pos = append(pos, po)
-			log.Trace("clean ok", "key", chunkHash, "gcKey", gcIdxKey, "gcData", gcIdxData)
 				okEntryCount++
 				if idx.Idx > poPtrs[po] {
 					poPtrs[po] = idx.Idx
 				}
 			}
 			totalEntryCount++
-		if err := s.db.Write(&batch); err != nil {
-			return err
-		}
 			it.Next()
 		}
-
 		it.Release()
+
+		err := s.db.Write(&batch)
+		if err != nil {
+			return err
+		}
+
+		for i, okIdx := range idxs {
+			gcIdxKey := getGCIdxKey(&okIdx)
+			gcIdxData := getGCIdxValue(&okIdx, pos[i], chunkHashes[i])
+			batch.Put(gcIdxKey, gcIdxData)
+			log.Trace("clean ok", "key", chunkHashes[i], "gcKey", gcIdxKey, "gcData", gcIdxData)
+		}
+		err = s.db.Write(&batch)
+		if err != nil {
+			return err
+		}
+
+	}
+
 	log.Debug("gc cleanup entries", "ok", okEntryCount, "total", totalEntryCount, "batchlen", batch.Len())
 
 	var entryCount [8]byte

swarm/storage/ldbstore_test.go

@@ -705,7 +705,7 @@ func TestCleanIndex(t *testing.T) {
 
 	// second gc index should still be fixed
 	if _, err := ldb.db.Get(gcSecondCorrectKey); err != nil {
-		t.Fatalf("expected gc 1 idx to be present: %v", idxKey)
+		t.Fatalf("expected gc 1 idx %v to be present: %v", gcSecondCorrectKey, idxKey)
 	}
 
 	// third gc index should be unchanged