forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 18:32:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

swarm/shed, swarm/storage/mock: add MockIndex to shed

Browse source
This commit is contained in:
Janos Guljas 2018-11-19 14:32:07 +01:00
parent ba8d0f0e50
commit 4696fb29c3
10 changed files with 393 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
swarm/shed/example_store_test.go
View file

@ -24,10 +24,14 @@ import (
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/shed"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
"github.com/syndtr/goleveldb/leveldb"
)
@ -43,7 +47,7 @@ type Store struct {
schemaName shed.StringField
sizeCounter shed.Uint64Field
accessCounter shed.Uint64Field
retrievalIndex shed.Index
retrievalIndex shed.IndexInterface // example of swapable Index/MockIndex
accessIndex shed.Index
gcIndex shed.Index
}
@ -51,7 +55,7 @@ type Store struct {
// New returns new Store. All fields and indexes are initialized
// and possible conflicts with schema from existing database is checked
// automatically.
func New(path string) (s *Store, err error) {
func New(path string, mockStore *mock.NodeStore) (s *Store, err error) {
db, err := shed.NewDB(path)
if err != nil {
return nil, err
@ -70,7 +74,7 @@ func New(path string) (s *Store, err error) {
return nil, err
}
// Index storing actual chunk address, data and store timestamp.
s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
retrievalIndex, err := db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
return fields.Address, nil
},
@ -90,6 +94,14 @@ func New(path string) (s *Store, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
s.retrievalIndex = retrievalIndex
if mockStore != nil {
// If mock store is provided, use it for retrieval index.
s.retrievalIndex = retrievalIndex.NewMockIndex(mockStore)
}
// Index storing access timestamp for a particular address.
// It is needed in order to update gc index keys for iteration order.
s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{
@ -110,6 +122,9 @@ func New(path string) (s *Store, err error) {
return e, nil
},
})
if err != nil {
return nil, err
}
// Index with keys ordered by access timestamp for garbage collection prioritization.
s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{
EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
@ -303,7 +318,20 @@ func Example_store() {
}
defer os.RemoveAll(dir)
s, err := New(dir)
var mockStore *mock.NodeStore
// Configuring mock store using environment variable is
// just an example, it can be accomplished in more elegant ways.
if strings.EqualFold(os.Getenv("SWARM_USE_MOCKSTORE"), "true") {
// Global store is constructed here for example purposes.
// It should be available globally so that all nodes can access it.
globalStore := mem.NewGlobalStore()
// An arbitrary address is used for this example.
// In real situations this address should be the same as the Swarm
// node address.
mockStore = globalStore.NewNodeStore(common.HexToAddress("12345678"))
}
s, err := New(dir, mockStore)
if err != nil {
log.Fatal(err)
}

79
swarm/shed/index.go
View file

@ -17,6 +17,7 @@
package shed
import (
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/syndtr/goleveldb/leveldb"
)
@ -64,6 +65,7 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) {
// - getting a particular IndexItem
// - saving a particular IndexItem
// - iterating over a sorted LevelDB keys
// It implements IndexIteratorInterface interface.
type Index struct {
db *DB
prefix []byte
@ -94,16 +96,8 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) {
return Index{
db: db,
prefix: prefix,
encodeKeyFunc: func(e IndexItem) (key []byte, err error) {
key, err = funcs.EncodeKey(e)
if err != nil {
return nil, err
}
return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil
},
decodeKeyFunc: func(key []byte) (e IndexItem, err error) {
return funcs.DecodeKey(key[1:])
},
encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id),
decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey),
encodeValueFunc: funcs.EncodeValue,
decodeValueFunc: funcs.DecodeValue,
}, nil
@ -248,3 +242,68 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) {
}
return it.Error()
}
// NewMockIndex is a helper function to easily construct MockIndex
// when from the same definition of Index.
func (f Index) NewMockIndex(store *mock.NodeStore) (m MockIndex) {
return MockIndex{
store: store,
prefix: f.prefix,
encodeKeyFunc: f.encodeKeyFunc,
decodeKeyFunc: f.decodeKeyFunc,
encodeValueFunc: f.encodeValueFunc,
decodeValueFunc: f.decodeValueFunc,
}
}
// IndexInterface defines methods that are required for a simple index
// that does not use iterations.
// It can be used when Index should be swapped with MockIndex or any other
// IndexInterface implementation.
// In most cases, interface is not needed and it is recommended to use the
// Index implementation whenever possible.
type IndexInterface interface {
Get(keyFields IndexItem) (out IndexItem, err error)
Put(i IndexItem) (err error)
PutInBatch(batch *leveldb.Batch, i IndexItem) (err error)
Delete(keyFields IndexItem) (err error)
DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error)
}
// IndexIteratorInterface defines metods for a full index implementation with
// iterators. Index type implements this type.
// In most cases, interface is not needed and it is recommended to use the
// Index implementation whenever possible.
type IndexIteratorInterface interface {
IndexInterface
IterateAll(fn IndexIterFunc) (err error)
IterateFrom(start IndexItem, fn IndexIterFunc) (err error)
}
// newIndexEncodeKeyFunc adjusts Index and MockIndex LevelDB key
// by appending the provided index id byte.
// This is needed to avoid collisions between keys of different
// indexes as all index ids are unique.
func newIndexEncodeKeyFunc(
encodeKeyFunc func(fields IndexItem) (key []byte, err error),
id byte,
) (f func(e IndexItem) (key []byte, err error)) {
prefix := []byte{id}
return func(e IndexItem) (key []byte, err error) {
key, err = encodeKeyFunc(e)
if err != nil {
return nil, err
}
return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil
}
}
// newDecodeKeyFunc reverses the newIndexEncodeKeyFunc constructd key
// to transparently work with index keys without their index ids.
// This function is used in NewIndex and NewMockIndex constructors.
// It assumes that index keys are prefixed with only one byte.
func newDecodeKeyFunc(decodeKeyFunc func(key []byte) (e IndexItem, err error)) (f func(key []byte) (e IndexItem, err error)) {
return func(key []byte) (e IndexItem, err error) {
return decodeKeyFunc(key[1:])
}
}

113
swarm/shed/index_mock.go Normal file
View file

@ -0,0 +1,113 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/syndtr/goleveldb/leveldb"
)
// MockIndex provides a way to inject a mock.NodeStore to store
// data centrally instead on provided DB. DB is used just for schema
// validation and identifying byte prefix for the particular index
// that is mocked.
// Iterator functions are not implemented and MockIndex can not replace
// indexes that rely on them.
// It implements IndexField interface.
type MockIndex struct {
store *mock.NodeStore
prefix []byte
encodeKeyFunc func(fields IndexItem) (key []byte, err error)
decodeKeyFunc func(key []byte) (e IndexItem, err error)
encodeValueFunc func(fields IndexItem) (value []byte, err error)
decodeValueFunc func(value []byte) (e IndexItem, err error)
}
// NewMockIndex returns a new MockIndex instance with defined name and
// encoding functions. The name must be unique and will be validated
// on database schema for a key prefix byte.
// The data will not be saved on the DB itself, but using a provided
// mock.NodeStore.
func (db *DB) NewMockIndex(store *mock.NodeStore, name string, funcs IndexFuncs) (f MockIndex, err error) {
id, err := db.schemaIndexPrefix(name)
if err != nil {
return f, err
}
return MockIndex{
store: store,
prefix: []byte{id},
encodeKeyFunc: newIndexEncodeKeyFunc(funcs.EncodeKey, id),
decodeKeyFunc: newDecodeKeyFunc(funcs.DecodeKey),
encodeValueFunc: funcs.EncodeValue,
decodeValueFunc: funcs.DecodeValue,
}, nil
}
// Get accepts key fields represented as IndexItem to retrieve a
// value from the index and return maximum available information
// from the index represented as another IndexItem.
func (f MockIndex) Get(keyFields IndexItem) (out IndexItem, err error) {
key, err := f.encodeKeyFunc(keyFields)
if err != nil {
return out, err
}
value, err := f.store.Get(key)
if err != nil {
return out, err
}
out, err = f.decodeValueFunc(value)
if err != nil {
return out, err
}
return out.Join(keyFields), nil
}
// Put accepts IndexItem to encode information from it
// and save it to the database.
func (f MockIndex) Put(i IndexItem) (err error) {
key, err := f.encodeKeyFunc(i)
if err != nil {
return err
}
value, err := f.encodeValueFunc(i)
if err != nil {
return err
}
return f.store.Put(key, value)
}
// PutInBatch is the same as Put method.
// Batch is ignored and the data is saved to the mock store instantly.
func (f MockIndex) PutInBatch(_ *leveldb.Batch, i IndexItem) (err error) {
return f.Put(i)
}
// Delete accepts IndexItem to remove a key/value pair
// form the database based on its fields.
func (f MockIndex) Delete(keyFields IndexItem) (err error) {
key, err := f.encodeKeyFunc(keyFields)
if err != nil {
return err
}
return f.store.Delete(key)
}
// DeleteInBatch is the same as Delete just the operation.
// Batch is ignored and the data is deleted on the mock store instantly.
func (f MockIndex) DeleteInBatch(_ *leveldb.Batch, keyFields IndexItem) (err error) {
return f.Delete(keyFields)
}

45
swarm/shed/index_mock_test.go Normal file
View file

@ -0,0 +1,45 @@
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package shed
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
)
// TestMockIndex validates put, get and delete functions of
// the MockIndex implementation.
func TestMockIndex(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
globalStore := mem.NewGlobalStore()
index, err := db.NewMockIndex(
globalStore.NewNodeStore(common.HexToAddress("12345678")),
"retrieval",
retrievalIndexFuncs,
)
if err != nil {
t.Fatal(err)
}
testIndex(t, db, index)
}

52
swarm/shed/index_test.go
View file

@ -24,6 +24,9 @@ import (
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/swarm/storage/mock"
"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
"github.com/syndtr/goleveldb/leveldb"
)
@ -49,7 +52,7 @@ var retrievalIndexFuncs = IndexFuncs{
},
}
// TestIndex validates put, get and delete functions of the index.
// TestIndex validates put, get and delete functions of the Index implementation.
func TestIndex(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
@ -59,6 +62,29 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
testIndex(t, db, index)
}
// TestIndex_NewMockIndex validates put, get and delete functions of
// the MockIndex implementation, when constructed with Index.NewMockIndex function.
func TestIndex_NewMockIndex(t *testing.T) {
db, cleanupFunc := newTestDB(t)
defer cleanupFunc()
index, err := db.NewIndex("retrieval", retrievalIndexFuncs)
if err != nil {
t.Fatal(err)
}
globalStore := mem.NewGlobalStore()
mockIndex := index.NewMockIndex(globalStore.NewNodeStore(common.HexToAddress("12345678")))
testIndex(t, db, mockIndex)
}
// testIndex validates put, get and delete functions of a index interface.
func testIndex(t *testing.T, db *DB, index IndexInterface) {
t.Run("put", func(t *testing.T) {
want := IndexItem{
Address: []byte("put-hash"),
@ -66,7 +92,7 @@ func TestIndex(t *testing.T) {
StoreTimestamp: time.Now().UTC().UnixNano(),
}
err = index.Put(want)
err := index.Put(want)
if err != nil {
t.Fatal(err)
}
@ -108,7 +134,7 @@ func TestIndex(t *testing.T) {
batch := new(leveldb.Batch)
index.PutInBatch(batch, want)
db.WriteBatch(batch)
err := db.WriteBatch(batch)
if err != nil {
t.Fatal(err)
}
@ -150,7 +176,7 @@ func TestIndex(t *testing.T) {
StoreTimestamp: time.Now().UTC().UnixNano(),
}
err = index.Put(want)
err := index.Put(want)
if err != nil {
t.Fatal(err)
}
@ -169,11 +195,15 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
wantErr := leveldb.ErrNotFound
if _, ok := index.(MockIndex); ok {
wantErr = mock.ErrNotFound
}
got, err = index.Get(IndexItem{
Address: want.Address,
})
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
if err != wantErr {
t.Fatalf("got error %v, want %v", err, wantErr)
}
})
@ -184,7 +214,7 @@ func TestIndex(t *testing.T) {
StoreTimestamp: time.Now().UTC().UnixNano(),
}
err = index.Put(want)
err := index.Put(want)
if err != nil {
t.Fatal(err)
}
@ -205,11 +235,15 @@ func TestIndex(t *testing.T) {
t.Fatal(err)
}
wantErr := leveldb.ErrNotFound
if _, ok := index.(MockIndex); ok {
wantErr = mock.ErrNotFound
}
got, err = index.Get(IndexItem{
Address: want.Address,
})
if err != leveldb.ErrNotFound {
t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
if err != wantErr {
t.Fatalf("got error %v, want %v", err, wantErr)
}
})
}

7
swarm/storage/mock/db/db.go
View file

@ -86,6 +86,13 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
return s.db.Write(batch, nil)
}
// Delete removes the chunk reference to node with address addr.
func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
batch := new(leveldb.Batch)
batch.Delete(nodeDBKey(addr, key))
return s.db.Write(batch, nil)
}
// HasKey returns whether a node with addr contains the key.
func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
has, err := s.db.Has(nodeDBKey(addr, key), nil)

16
swarm/storage/mock/mem/mem.go
View file

@ -83,6 +83,22 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
return nil
}
// Delete removes the chunk data for node with address addr.
func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
var count int
if _, ok := s.nodes[string(key)]; ok {
delete(s.nodes[string(key)], addr)
count = len(s.nodes[string(key)])
}
if count == 0 {
delete(s.data, string(key))
}
return nil
}
// HasKey returns whether a node with addr contains the key.
func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
s.mu.Lock()

7
swarm/storage/mock/mock.go
View file

@ -70,6 +70,12 @@ func (n *NodeStore) Put(key []byte, data []byte) error {
return n.store.Put(n.addr, key, data)
}
// Delete removes chunk data for a key for a node that has the address
// provided on NodeStore initialization.
func (n *NodeStore) Delete(key []byte) error {
return n.store.Delete(n.addr, key)
}
// GlobalStorer defines methods for mock db store
// that stores chunk data for all swarm nodes.
// It is used in tests to construct mock NodeStores
@ -77,6 +83,7 @@ func (n *NodeStore) Put(key []byte, data []byte) error {
type GlobalStorer interface {
Get(addr common.Address, key []byte) (data []byte, err error)
Put(addr common.Address, key []byte, data []byte) error
Delete(addr common.Address, key []byte) error
HasKey(addr common.Address, key []byte) bool
// NewNodeStore creates an instance of NodeStore
// to be used by a single swarm node with

6
swarm/storage/mock/rpc/rpc.go
View file

@ -73,6 +73,12 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
return err
}
// Delete calls a Delete method to RPC server.
func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
err := s.client.Call(nil, "mockStore_delete", addr, key)
return err
}
// HasKey calls a HasKey method to RPC server.
func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
var has bool

53
swarm/storage/mock/test/test.go
View file

@ -72,6 +72,31 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
}
}
}
t.Run("delete", func(t *testing.T) {
chunkAddr := storage.Address([]byte("1234567890abcd"))
for _, addr := range addrs {
err := globalStore.Put(addr, chunkAddr, []byte("data"))
if err != nil {
t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
}
}
firstNodeAddr := addrs[0]
if err := globalStore.Delete(firstNodeAddr, chunkAddr); err != nil {
t.Fatalf("delete from store %s key %s: %v", firstNodeAddr.Hex(), chunkAddr.Hex(), err)
}
for i, addr := range addrs {
_, err := globalStore.Get(addr, chunkAddr)
if i == 0 {
if err != mock.ErrNotFound {
t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err)
}
} else {
if err != nil {
t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
}
}
}
})
})
t.Run("NodeStore", func(t *testing.T) {
@ -114,6 +139,34 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
}
}
}
t.Run("delete", func(t *testing.T) {
chunkAddr := storage.Address([]byte("1234567890abcd"))
var chosenStore *mock.NodeStore
for addr, store := range nodes {
if chosenStore == nil {
chosenStore = store
}
err := store.Put(chunkAddr, []byte("data"))
if err != nil {
t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
}
}
if err := chosenStore.Delete(chunkAddr); err != nil {
t.Fatalf("delete key %s: %v", chunkAddr.Hex(), err)
}
for addr, store := range nodes {
_, err := store.Get(chunkAddr)
if store == chosenStore {
if err != mock.ErrNotFound {
t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err)
}
} else {
if err != nil {
t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
}
}
}
})
})
}