swarm/storage/localstore: add TestModeRemoval

swarm/storage/localstore/localstore.go

@@ -235,7 +235,6 @@ func New(path string, baseKey []byte, opts ...Option) (db *DB, err error) {
 			return nil, nil
 		},
 		DecodeValue: func(value []byte) (e shed.IndexItem, err error) {
-			e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
 			return e, nil
 		},
 	})

swarm/storage/localstore/mode.go

@@ -81,7 +81,6 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er
 	switch mode {
 	case ModeRequest, modeAccess:
 		// update the access counter
-		// Q: can we do this asynchronously
 		return out, db.update(context.TODO(), mode, out)
 	default:
 		// all other modes are not updating the index
@@ -89,7 +88,8 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er
 	return out, nil
 }
 
-// update is called by an Accessor with a specific Mode.
+// update is called by an Accessor with a specific Mode,
+// and also in access for updating access timestamp and gc index.
 // This function calls updateBatch to perform operations
 // on indexes and fields within a single batch.
 func (db *DB) update(ctx context.Context, mode Mode, item shed.IndexItem) error {
@@ -146,8 +146,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
 	case ModeSyncing:
 		// put to indexes: retrieve, pull
 		item.StoreTimestamp = now()
-		item.AccessTimestamp = now()
 		if db.useRetrievalCompositeIndex {
+			item.AccessTimestamp = now()
 			db.retrievalCompositeIndex.PutInBatch(b.Batch, item)
 		} else {
 			db.retrievalDataIndex.PutInBatch(b.Batch, item)
@@ -158,8 +158,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
 	case ModeUpload:
 		// put to indexes: retrieve, push, pull
 		item.StoreTimestamp = now()
-		item.AccessTimestamp = now()
 		if db.useRetrievalCompositeIndex {
+			item.AccessTimestamp = now()
 			db.retrievalCompositeIndex.PutInBatch(b.Batch, item)
 		} else {
 			db.retrievalDataIndex.PutInBatch(b.Batch, item)
@@ -254,24 +254,25 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
 		// a property of a chunk provided to Accessor.Put.
 		if db.useRetrievalCompositeIndex {
 			i, err := db.retrievalCompositeIndex.Get(item)
-			switch err {
+			if err != nil {
-			case nil:
-				item.AccessTimestamp = i.AccessTimestamp
-			case leveldb.ErrNotFound:
-				item.AccessTimestamp = now()
-			default:
 				return err
 			}
+			item.StoreTimestamp = i.StoreTimestamp
+			item.AccessTimestamp = i.AccessTimestamp
 		} else {
 			i, err := db.retrievalAccessIndex.Get(item)
 			switch err {
 			case nil:
 				item.AccessTimestamp = i.AccessTimestamp
 			case leveldb.ErrNotFound:
-				item.AccessTimestamp = now()
 			default:
 				return err
 			}
+			i, err = db.retrievalDataIndex.Get(item)
+			if err != nil {
+				return err
+			}
+			item.StoreTimestamp = i.StoreTimestamp
 		}
 		if db.useRetrievalCompositeIndex {
 			db.retrievalCompositeIndex.DeleteInBatch(b.Batch, item)

swarm/storage/localstore/mode_test.go

@@ -73,7 +73,7 @@ func testModeSyncingValues(t *testing.T, db *DB) {
 
 	t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp))
 
-	t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp))
+	t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil))
 
 	t.Run("size counter", testSizeCounter(db, wantSize))
 }
@@ -123,9 +123,9 @@ func testModeUploadValues(t *testing.T, db *DB) {
 
 	t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp))
 
-	t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp))
+	t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil))
 
-	t.Run("push index", testPullIndexValues(db, chunk, wantTimestamp))
+	t.Run("push index", testPushIndexValues(db, chunk, wantTimestamp, nil))
 
 	t.Run("size counter", testSizeCounter(db, wantSize))
 }
@@ -277,7 +277,7 @@ func testModeAccessValues(t *testing.T, db *DB) {
 
 		t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, uploadTimestamp))
 
-		t.Run("gc index count", testGCIndexCount(db, 1))
+		t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1))
 	})
 
 	t.Run("second get", func(t *testing.T) {
@@ -303,10 +303,87 @@ func testModeAccessValues(t *testing.T, db *DB) {
 
 		t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, accessTimestamp))
 
-		t.Run("gc index count", testGCIndexCount(db, 1))
+		t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1))
 	})
 }
 
+// TestModeRemoval validates internal data operations and state
+// for ModeRemoval on DB with default configuration.
+func TestModeRemoval(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	testModeRemovalValues(t, db)
+}
+
+// TestModeRemoval_withRetrievalCompositeIndex validates internal
+// data operations and state for ModeRemoval on DB with
+// retrieval composite index enabled.
+func TestModeRemoval_withRetrievalCompositeIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t, WithRetrievalCompositeIndex(true))
+	defer cleanupFunc()
+
+	testModeRemovalValues(t, db)
+}
+
+// testModeRemovalValues validates ModeRemoval index values on the provided DB.
+func testModeRemovalValues(t *testing.T, db *DB) {
+	a := db.Accessor(ModeUpload)
+
+	chunk := generateRandomChunk()
+
+	err := a.Put(context.Background(), chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	a = db.Accessor(modeRemoval)
+
+	wantSize, err := db.sizeCounter.Get()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wantSize--
+
+	err = a.Put(context.Background(), chunk)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Run("retrieve indexes", func(t *testing.T) {
+		wantErr := leveldb.ErrNotFound
+		if db.useRetrievalCompositeIndex {
+			_, err := db.retrievalCompositeIndex.Get(addressToItem(chunk.Address()))
+			if err != wantErr {
+				t.Errorf("got error %v, want %v", err, wantErr)
+			}
+			t.Run("retrieve index count", testIndexItemsCount(db.retrievalCompositeIndex, 0))
+		} else {
+			_, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
+			if err != wantErr {
+				t.Errorf("got error %v, want %v", err, wantErr)
+			}
+			t.Run("retrieve data index count", testIndexItemsCount(db.retrievalDataIndex, 0))
+
+			// access index should not be set
+			_, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address()))
+			if err != wantErr {
+				t.Errorf("got error %v, want %v", err, wantErr)
+			}
+			t.Run("retrieve access index count", testIndexItemsCount(db.retrievalAccessIndex, 0))
+		}
+	})
+
+	t.Run("pull index", testPullIndexValues(db, chunk, 0, leveldb.ErrNotFound))
+
+	t.Run("pull index count", testIndexItemsCount(db.pullIndex, 0))
+
+	t.Run("gc index count", testIndexItemsCount(db.gcIndex, 0))
+
+	t.Run("size counter", testSizeCounter(db, wantSize))
+}
+
 // testRetrieveIndexesValues returns a test function that validates if the right
 // chunk values are in the retrieval indexes.
 func testRetrieveIndexesValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
@@ -363,16 +440,18 @@ func testRetrieveIndexesValuesWithAccess(db *DB, chunk storage.Chunk, storeTimes
 
 // testPullIndexValues returns a test function that validates if the right
 // chunk values are in the pull index.
-func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64) func(t *testing.T) {
+func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
 	return func(t *testing.T) {
 		item, err := db.pullIndex.Get(shed.IndexItem{
 			Address:        chunk.Address(),
 			StoreTimestamp: storeTimestamp,
 		})
-		if err != nil {
+		if err != wantError {
-			t.Fatal(err)
+			t.Errorf("got error %v, want %v", err, wantError)
+		}
+		if err == nil {
+			validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
 		}
-		validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
 	}
 }
 
@@ -409,17 +488,17 @@ func testGCIndexValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimest
 	}
 }
 
-// testGCIndexCount returns a test function that validates if
+// testIndexItemsCount returns a test function that validates if
-// gc index contains expected number of key/value pairs.
+// an index contains expected number of key/value pairs.
-func testGCIndexCount(db *DB, want int) func(t *testing.T) {
+func testIndexItemsCount(i shed.Index, want int) func(t *testing.T) {
 	return func(t *testing.T) {
 		var c int
-		db.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) {
+		i.IterateAll(func(item shed.IndexItem) (stop bool, err error) {
 			c++
 			return
 		})
 		if c != want {
-			t.Errorf("got %v item in gc index, want %v", c, want)
+			t.Errorf("got %v items in index, want %v", c, want)
 		}
 	}
 }