diff --git a/swarm/storage/localstore/localstore.go b/swarm/storage/localstore/localstore.go index e8af232964..6de38310c4 100644 --- a/swarm/storage/localstore/localstore.go +++ b/swarm/storage/localstore/localstore.go @@ -235,7 +235,6 @@ func New(path string, baseKey []byte, opts ...Option) (db *DB, err error) { return nil, nil }, DecodeValue: func(value []byte) (e shed.IndexItem, err error) { - e.AccessTimestamp = int64(binary.BigEndian.Uint64(value)) return e, nil }, }) diff --git a/swarm/storage/localstore/mode.go b/swarm/storage/localstore/mode.go index dbebe91333..3bb95f889f 100644 --- a/swarm/storage/localstore/mode.go +++ b/swarm/storage/localstore/mode.go @@ -81,7 +81,6 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er switch mode { case ModeRequest, modeAccess: // update the access counter - // Q: can we do this asynchronously return out, db.update(context.TODO(), mode, out) default: // all other modes are not updating the index @@ -89,7 +88,8 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er return out, nil } -// update is called by an Accessor with a specific Mode. +// update is called by an Accessor with a specific Mode, +// and also in access for updating access timestamp and gc index. // This function calls updateBatch to perform operations // on indexes and fields within a single batch. func (db *DB) update(ctx context.Context, mode Mode, item shed.IndexItem) error { @@ -146,8 +146,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error) case ModeSyncing: // put to indexes: retrieve, pull item.StoreTimestamp = now() - item.AccessTimestamp = now() if db.useRetrievalCompositeIndex { + item.AccessTimestamp = now() db.retrievalCompositeIndex.PutInBatch(b.Batch, item) } else { db.retrievalDataIndex.PutInBatch(b.Batch, item) @@ -158,8 +158,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error) case ModeUpload: // put to indexes: retrieve, push, pull item.StoreTimestamp = now() - item.AccessTimestamp = now() if db.useRetrievalCompositeIndex { + item.AccessTimestamp = now() db.retrievalCompositeIndex.PutInBatch(b.Batch, item) } else { db.retrievalDataIndex.PutInBatch(b.Batch, item) @@ -254,24 +254,25 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error) // a property of a chunk provided to Accessor.Put. if db.useRetrievalCompositeIndex { i, err := db.retrievalCompositeIndex.Get(item) - switch err { - case nil: - item.AccessTimestamp = i.AccessTimestamp - case leveldb.ErrNotFound: - item.AccessTimestamp = now() - default: + if err != nil { return err } + item.StoreTimestamp = i.StoreTimestamp + item.AccessTimestamp = i.AccessTimestamp } else { i, err := db.retrievalAccessIndex.Get(item) switch err { case nil: item.AccessTimestamp = i.AccessTimestamp case leveldb.ErrNotFound: - item.AccessTimestamp = now() default: return err } + i, err = db.retrievalDataIndex.Get(item) + if err != nil { + return err + } + item.StoreTimestamp = i.StoreTimestamp } if db.useRetrievalCompositeIndex { db.retrievalCompositeIndex.DeleteInBatch(b.Batch, item) diff --git a/swarm/storage/localstore/mode_test.go b/swarm/storage/localstore/mode_test.go index 50a9db7938..7321ccdfda 100644 --- a/swarm/storage/localstore/mode_test.go +++ b/swarm/storage/localstore/mode_test.go @@ -73,7 +73,7 @@ func testModeSyncingValues(t *testing.T, db *DB) { t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp)) - t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp)) + t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil)) t.Run("size counter", testSizeCounter(db, wantSize)) } @@ -123,9 +123,9 @@ func testModeUploadValues(t *testing.T, db *DB) { t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp)) - t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp)) + t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil)) - t.Run("push index", testPullIndexValues(db, chunk, wantTimestamp)) + t.Run("push index", testPushIndexValues(db, chunk, wantTimestamp, nil)) t.Run("size counter", testSizeCounter(db, wantSize)) } @@ -277,7 +277,7 @@ func testModeAccessValues(t *testing.T, db *DB) { t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, uploadTimestamp)) - t.Run("gc index count", testGCIndexCount(db, 1)) + t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1)) }) t.Run("second get", func(t *testing.T) { @@ -303,10 +303,87 @@ func testModeAccessValues(t *testing.T, db *DB) { t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, accessTimestamp)) - t.Run("gc index count", testGCIndexCount(db, 1)) + t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1)) }) } +// TestModeRemoval validates internal data operations and state +// for ModeRemoval on DB with default configuration. +func TestModeRemoval(t *testing.T) { + db, cleanupFunc := newTestDB(t) + defer cleanupFunc() + + testModeRemovalValues(t, db) +} + +// TestModeRemoval_withRetrievalCompositeIndex validates internal +// data operations and state for ModeRemoval on DB with +// retrieval composite index enabled. +func TestModeRemoval_withRetrievalCompositeIndex(t *testing.T) { + db, cleanupFunc := newTestDB(t, WithRetrievalCompositeIndex(true)) + defer cleanupFunc() + + testModeRemovalValues(t, db) +} + +// testModeRemovalValues validates ModeRemoval index values on the provided DB. +func testModeRemovalValues(t *testing.T, db *DB) { + a := db.Accessor(ModeUpload) + + chunk := generateRandomChunk() + + err := a.Put(context.Background(), chunk) + if err != nil { + t.Fatal(err) + } + + a = db.Accessor(modeRemoval) + + wantSize, err := db.sizeCounter.Get() + if err != nil { + t.Fatal(err) + } + + wantSize-- + + err = a.Put(context.Background(), chunk) + if err != nil { + t.Fatal(err) + } + + t.Run("retrieve indexes", func(t *testing.T) { + wantErr := leveldb.ErrNotFound + if db.useRetrievalCompositeIndex { + _, err := db.retrievalCompositeIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + t.Run("retrieve index count", testIndexItemsCount(db.retrievalCompositeIndex, 0)) + } else { + _, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + t.Run("retrieve data index count", testIndexItemsCount(db.retrievalDataIndex, 0)) + + // access index should not be set + _, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address())) + if err != wantErr { + t.Errorf("got error %v, want %v", err, wantErr) + } + t.Run("retrieve access index count", testIndexItemsCount(db.retrievalAccessIndex, 0)) + } + }) + + t.Run("pull index", testPullIndexValues(db, chunk, 0, leveldb.ErrNotFound)) + + t.Run("pull index count", testIndexItemsCount(db.pullIndex, 0)) + + t.Run("gc index count", testIndexItemsCount(db.gcIndex, 0)) + + t.Run("size counter", testSizeCounter(db, wantSize)) +} + // testRetrieveIndexesValues returns a test function that validates if the right // chunk values are in the retrieval indexes. func testRetrieveIndexesValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) { @@ -363,16 +440,18 @@ func testRetrieveIndexesValuesWithAccess(db *DB, chunk storage.Chunk, storeTimes // testPullIndexValues returns a test function that validates if the right // chunk values are in the pull index. -func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64) func(t *testing.T) { +func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) { return func(t *testing.T) { item, err := db.pullIndex.Get(shed.IndexItem{ Address: chunk.Address(), StoreTimestamp: storeTimestamp, }) - if err != nil { - t.Fatal(err) + if err != wantError { + t.Errorf("got error %v, want %v", err, wantError) + } + if err == nil { + validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0) } - validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0) } } @@ -409,17 +488,17 @@ func testGCIndexValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimest } } -// testGCIndexCount returns a test function that validates if -// gc index contains expected number of key/value pairs. -func testGCIndexCount(db *DB, want int) func(t *testing.T) { +// testIndexItemsCount returns a test function that validates if +// an index contains expected number of key/value pairs. +func testIndexItemsCount(i shed.Index, want int) func(t *testing.T) { return func(t *testing.T) { var c int - db.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) { + i.IterateAll(func(item shed.IndexItem) (stop bool, err error) { c++ return }) if c != want { - t.Errorf("got %v item in gc index, want %v", c, want) + t.Errorf("got %v items in index, want %v", c, want) } } }