forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 18:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

swarm/storage/localstore: add TestModeRemoval

Browse source
This commit is contained in:
Janos Guljas 2018-12-04 14:56:44 +01:00
parent af1b137226
commit 96409ff600
3 changed files with 105 additions and 26 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
swarm/storage/localstore/localstore.go
View file

@ -235,7 +235,6 @@ func New(path string, baseKey []byte, opts ...Option) (db *DB, err error) {
return nil, nil
},
DecodeValue: func(value []byte) (e shed.IndexItem, err error) {
e.AccessTimestamp = int64(binary.BigEndian.Uint64(value))
return e, nil
},
})

23
swarm/storage/localstore/mode.go
View file

@ -81,7 +81,6 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er
switch mode {
case ModeRequest, modeAccess:
// update the access counter
// Q: can we do this asynchronously
return out, db.update(context.TODO(), mode, out)
default:
// all other modes are not updating the index
@ -89,7 +88,8 @@ func (db *DB) access(mode Mode, item shed.IndexItem) (out shed.IndexItem, err er
return out, nil
}
// update is called by an Accessor with a specific Mode.
// update is called by an Accessor with a specific Mode,
// and also in access for updating access timestamp and gc index.
// This function calls updateBatch to perform operations
// on indexes and fields within a single batch.
func (db *DB) update(ctx context.Context, mode Mode, item shed.IndexItem) error {
@ -146,8 +146,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
case ModeSyncing:
// put to indexes: retrieve, pull
item.StoreTimestamp = now()
item.AccessTimestamp = now()
if db.useRetrievalCompositeIndex {
item.AccessTimestamp = now()
db.retrievalCompositeIndex.PutInBatch(b.Batch, item)
} else {
db.retrievalDataIndex.PutInBatch(b.Batch, item)
@ -158,8 +158,8 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
case ModeUpload:
// put to indexes: retrieve, push, pull
item.StoreTimestamp = now()
item.AccessTimestamp = now()
if db.useRetrievalCompositeIndex {
item.AccessTimestamp = now()
db.retrievalCompositeIndex.PutInBatch(b.Batch, item)
} else {
db.retrievalDataIndex.PutInBatch(b.Batch, item)
@ -254,24 +254,25 @@ func (db *DB) updateBatch(b *batch, mode Mode, item shed.IndexItem) (err error)
// a property of a chunk provided to Accessor.Put.
if db.useRetrievalCompositeIndex {
i, err := db.retrievalCompositeIndex.Get(item)
switch err {
case nil:
item.AccessTimestamp = i.AccessTimestamp
case leveldb.ErrNotFound:
item.AccessTimestamp = now()
default:
if err != nil {
return err
}
item.StoreTimestamp = i.StoreTimestamp
item.AccessTimestamp = i.AccessTimestamp
} else {
i, err := db.retrievalAccessIndex.Get(item)
switch err {
case nil:
item.AccessTimestamp = i.AccessTimestamp
case leveldb.ErrNotFound:
item.AccessTimestamp = now()
default:
return err
}
i, err = db.retrievalDataIndex.Get(item)
if err != nil {
return err
}
item.StoreTimestamp = i.StoreTimestamp
}
if db.useRetrievalCompositeIndex {
db.retrievalCompositeIndex.DeleteInBatch(b.Batch, item)

107
swarm/storage/localstore/mode_test.go
View file

@ -73,7 +73,7 @@ func testModeSyncingValues(t *testing.T, db *DB) {
t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp))
t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp))
t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil))
t.Run("size counter", testSizeCounter(db, wantSize))
}
@ -123,9 +123,9 @@ func testModeUploadValues(t *testing.T, db *DB) {
t.Run("retrieve indexes", testRetrieveIndexesValues(db, chunk, wantTimestamp, wantTimestamp))
t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp))
t.Run("pull index", testPullIndexValues(db, chunk, wantTimestamp, nil))
t.Run("push index", testPullIndexValues(db, chunk, wantTimestamp))
t.Run("push index", testPushIndexValues(db, chunk, wantTimestamp, nil))
t.Run("size counter", testSizeCounter(db, wantSize))
}
@ -277,7 +277,7 @@ func testModeAccessValues(t *testing.T, db *DB) {
t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, uploadTimestamp))
t.Run("gc index count", testGCIndexCount(db, 1))
t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1))
})
t.Run("second get", func(t *testing.T) {
@ -303,10 +303,87 @@ func testModeAccessValues(t *testing.T, db *DB) {
t.Run("gc index", testGCIndexValues(db, chunk, uploadTimestamp, accessTimestamp))
t.Run("gc index count", testGCIndexCount(db, 1))
t.Run("gc index count", testIndexItemsCount(db.gcIndex, 1))
})
}
// TestModeRemoval validates internal data operations and state
// for ModeRemoval on DB with default configuration.
func TestModeRemoval(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
testModeRemovalValues(t, db)
}
// TestModeRemoval_withRetrievalCompositeIndex validates internal
// data operations and state for ModeRemoval on DB with
// retrieval composite index enabled.
func TestModeRemoval_withRetrievalCompositeIndex(t *testing.T) {
db, cleanupFunc := newTestDB(t, WithRetrievalCompositeIndex(true))
defer cleanupFunc()
testModeRemovalValues(t, db)
}
// testModeRemovalValues validates ModeRemoval index values on the provided DB.
func testModeRemovalValues(t *testing.T, db *DB) {
a := db.Accessor(ModeUpload)
chunk := generateRandomChunk()
err := a.Put(context.Background(), chunk)
if err != nil {
t.Fatal(err)
}
a = db.Accessor(modeRemoval)
wantSize, err := db.sizeCounter.Get()
if err != nil {
t.Fatal(err)
}
wantSize--
err = a.Put(context.Background(), chunk)
if err != nil {
t.Fatal(err)
}
t.Run("retrieve indexes", func(t *testing.T) {
wantErr := leveldb.ErrNotFound
if db.useRetrievalCompositeIndex {
_, err := db.retrievalCompositeIndex.Get(addressToItem(chunk.Address()))
if err != wantErr {
t.Errorf("got error %v, want %v", err, wantErr)
}
t.Run("retrieve index count", testIndexItemsCount(db.retrievalCompositeIndex, 0))
} else {
_, err := db.retrievalDataIndex.Get(addressToItem(chunk.Address()))
if err != wantErr {
t.Errorf("got error %v, want %v", err, wantErr)
}
t.Run("retrieve data index count", testIndexItemsCount(db.retrievalDataIndex, 0))
// access index should not be set
_, err = db.retrievalAccessIndex.Get(addressToItem(chunk.Address()))
if err != wantErr {
t.Errorf("got error %v, want %v", err, wantErr)
}
t.Run("retrieve access index count", testIndexItemsCount(db.retrievalAccessIndex, 0))
}
})
t.Run("pull index", testPullIndexValues(db, chunk, 0, leveldb.ErrNotFound))
t.Run("pull index count", testIndexItemsCount(db.pullIndex, 0))
t.Run("gc index count", testIndexItemsCount(db.gcIndex, 0))
t.Run("size counter", testSizeCounter(db, wantSize))
}
// testRetrieveIndexesValues returns a test function that validates if the right
// chunk values are in the retrieval indexes.
func testRetrieveIndexesValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimestamp int64) func(t *testing.T) {
@ -363,16 +440,18 @@ func testRetrieveIndexesValuesWithAccess(db *DB, chunk storage.Chunk, storeTimes
// testPullIndexValues returns a test function that validates if the right
// chunk values are in the pull index.
func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64) func(t *testing.T) {
func testPullIndexValues(db *DB, chunk storage.Chunk, storeTimestamp int64, wantError error) func(t *testing.T) {
return func(t *testing.T) {
item, err := db.pullIndex.Get(shed.IndexItem{
Address: chunk.Address(),
StoreTimestamp: storeTimestamp,
})
if err != nil {
t.Fatal(err)
if err != wantError {
t.Errorf("got error %v, want %v", err, wantError)
}
if err == nil {
validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
}
validateItem(t, item, chunk.Address(), nil, storeTimestamp, 0)
}
}
@ -409,17 +488,17 @@ func testGCIndexValues(db *DB, chunk storage.Chunk, storeTimestamp, accessTimest
}
}
// testGCIndexCount returns a test function that validates if
// gc index contains expected number of key/value pairs.
func testGCIndexCount(db *DB, want int) func(t *testing.T) {
// testIndexItemsCount returns a test function that validates if
// an index contains expected number of key/value pairs.
func testIndexItemsCount(i shed.Index, want int) func(t *testing.T) {
return func(t *testing.T) {
var c int
db.gcIndex.IterateAll(func(item shed.IndexItem) (stop bool, err error) {
i.IterateAll(func(item shed.IndexItem) (stop bool, err error) {
c++
return
})
if c != want {
t.Errorf("got %v item in gc index, want %v", c, want)
t.Errorf("got %v items in index, want %v", c, want)
}
}
}