swarm/shed, swarm/storage/mock: add MockIndex to shed

swarm/shed/example_store_test.go

@@ -24,10 +24,14 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/swarm/shed"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -43,7 +47,7 @@ type Store struct {
 	schemaName     shed.StringField
 	sizeCounter    shed.Uint64Field
 	accessCounter  shed.Uint64Field
-	retrievalIndex shed.Index
+	retrievalIndex shed.IndexInterface // example of swapable Index/MockIndex
 	accessIndex    shed.Index
 	gcIndex        shed.Index
 }
@@ -51,7 +55,7 @@ type Store struct {
 // New returns new Store. All fields and indexes are initialized
 // and possible conflicts with schema from existing database is checked
 // automatically.
-func New(path string) (s *Store, err error) {
+func New(path string, mockStore *mock.NodeStore) (s *Store, err error) {
 	db, err := shed.NewDB(path)
 	if err != nil {
 		return nil, err
@@ -70,7 +74,7 @@ func New(path string) (s *Store, err error) {
 		return nil, err
 	}
 	// Index storing actual chunk address, data and store timestamp.
-	s.retrievalIndex, err = db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
+	retrievalIndex, err := db.NewIndex("Address->StoreTimestamp|Data", shed.IndexFuncs{
 		EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
 			return fields.Address, nil
 		},
@@ -90,6 +94,14 @@ func New(path string) (s *Store, err error) {
 			return e, nil
 		},
 	})
+	if err != nil {
+		return nil, err
+	}
+	s.retrievalIndex = retrievalIndex
+	if mockStore != nil {
+		// If mock store is provided, use it for retrieval index.
+		s.retrievalIndex = retrievalIndex.NewMockIndex(mockStore)
+	}
 	// Index storing access timestamp for a particular address.
 	// It is needed in order to update gc index keys for iteration order.
 	s.accessIndex, err = db.NewIndex("Address->AccessTimestamp", shed.IndexFuncs{
@@ -110,6 +122,9 @@ func New(path string) (s *Store, err error) {
 			return e, nil
 		},
 	})
+	if err != nil {
+		return nil, err
+	}
 	// Index with keys ordered by access timestamp for garbage collection prioritization.
 	s.gcIndex, err = db.NewIndex("AccessTimestamp|StoredTimestamp|Address->nil", shed.IndexFuncs{
 		EncodeKey: func(fields shed.IndexItem) (key []byte, err error) {
@@ -303,7 +318,20 @@ func Example_store() {
 	}
 	defer os.RemoveAll(dir)
 
-	s, err := New(dir)
+	var mockStore *mock.NodeStore
+	// Configuring mock store using environment variable is
+	// just an example, it can be accomplished in more elegant ways.
+	if strings.EqualFold(os.Getenv("SWARM_USE_MOCKSTORE"), "true") {
+		// Global store is constructed here for example purposes.
+		// It should be available globally so that all nodes can access it.
+		globalStore := mem.NewGlobalStore()
+		// An arbitrary address is used for this example.
+		// In real situations this address should be the same as the Swarm
+		// node address.
+		mockStore = globalStore.NewNodeStore(common.HexToAddress("12345678"))
+	}
+
+	s, err := New(dir, mockStore)
 	if err != nil {
 		log.Fatal(err)
 	}

swarm/shed/index.go

@@ -17,6 +17,7 @@
 package shed
 
 import (
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -64,6 +65,7 @@ func (i IndexItem) Join(i2 IndexItem) (new IndexItem) {
 // - getting a particular IndexItem
 // - saving a particular IndexItem
 // - iterating over a sorted LevelDB keys
+// It implements IndexIteratorInterface interface.
 type Index struct {
 	db              *DB
 	prefix          []byte
@@ -92,18 +94,10 @@ func (db *DB) NewIndex(name string, funcs IndexFuncs) (f Index, err error) {
 	}
 	prefix := []byte{id}
 	return Index{
-		db:     db,
+		db:              db,
-		prefix: prefix,
+		prefix:          prefix,
-		encodeKeyFunc: func(e IndexItem) (key []byte, err error) {
+		encodeKeyFunc:   newIndexEncodeKeyFunc(funcs.EncodeKey, id),
-			key, err = funcs.EncodeKey(e)
+		decodeKeyFunc:   newDecodeKeyFunc(funcs.DecodeKey),
-			if err != nil {
-				return nil, err
-			}
-			return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil
-		},
-		decodeKeyFunc: func(key []byte) (e IndexItem, err error) {
-			return funcs.DecodeKey(key[1:])
-		},
 		encodeValueFunc: funcs.EncodeValue,
 		decodeValueFunc: funcs.DecodeValue,
 	}, nil
@@ -248,3 +242,68 @@ func (f Index) IterateFrom(start IndexItem, fn IndexIterFunc) (err error) {
 	}
 	return it.Error()
 }
+
+// NewMockIndex is a helper function to easily construct MockIndex
+// when from the same definition of Index.
+func (f Index) NewMockIndex(store *mock.NodeStore) (m MockIndex) {
+	return MockIndex{
+		store:           store,
+		prefix:          f.prefix,
+		encodeKeyFunc:   f.encodeKeyFunc,
+		decodeKeyFunc:   f.decodeKeyFunc,
+		encodeValueFunc: f.encodeValueFunc,
+		decodeValueFunc: f.decodeValueFunc,
+	}
+}
+
+// IndexInterface defines methods that are required for a simple index
+// that does not use iterations.
+// It can be used when Index should be swapped with MockIndex or any other
+// IndexInterface implementation.
+// In most cases, interface is not needed and it is recommended to use the
+// Index implementation whenever possible.
+type IndexInterface interface {
+	Get(keyFields IndexItem) (out IndexItem, err error)
+	Put(i IndexItem) (err error)
+	PutInBatch(batch *leveldb.Batch, i IndexItem) (err error)
+	Delete(keyFields IndexItem) (err error)
+	DeleteInBatch(batch *leveldb.Batch, keyFields IndexItem) (err error)
+}
+
+// IndexIteratorInterface defines metods for a full index implementation with
+// iterators. Index type implements this type.
+// In most cases, interface is not needed and it is recommended to use the
+// Index implementation whenever possible.
+type IndexIteratorInterface interface {
+	IndexInterface
+	IterateAll(fn IndexIterFunc) (err error)
+	IterateFrom(start IndexItem, fn IndexIterFunc) (err error)
+}
+
+// newIndexEncodeKeyFunc adjusts Index and MockIndex LevelDB key
+// by appending the provided index id byte.
+// This is needed to avoid collisions between keys of different
+// indexes as all index ids are unique.
+func newIndexEncodeKeyFunc(
+	encodeKeyFunc func(fields IndexItem) (key []byte, err error),
+	id byte,
+) (f func(e IndexItem) (key []byte, err error)) {
+	prefix := []byte{id}
+	return func(e IndexItem) (key []byte, err error) {
+		key, err = encodeKeyFunc(e)
+		if err != nil {
+			return nil, err
+		}
+		return append(append(make([]byte, 0, len(key)+1), prefix...), key...), nil
+	}
+}
+
+// newDecodeKeyFunc reverses the newIndexEncodeKeyFunc constructd key
+// to transparently work with index keys without their index ids.
+// This function is used in NewIndex and NewMockIndex constructors.
+// It assumes that index keys are prefixed with only one byte.
+func newDecodeKeyFunc(decodeKeyFunc func(key []byte) (e IndexItem, err error)) (f func(key []byte) (e IndexItem, err error)) {
+	return func(key []byte) (e IndexItem, err error) {
+		return decodeKeyFunc(key[1:])
+	}
+}

swarm/shed/index_mock.go

@@ -0,0 +1,113 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package shed
+
+import (
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+// MockIndex provides a way to inject a mock.NodeStore to store
+// data centrally instead on provided DB. DB is used just for schema
+// validation and identifying byte prefix for the particular index
+// that is mocked.
+// Iterator functions are not implemented and MockIndex can not replace
+// indexes that rely on them.
+// It implements IndexField interface.
+type MockIndex struct {
+	store           *mock.NodeStore
+	prefix          []byte
+	encodeKeyFunc   func(fields IndexItem) (key []byte, err error)
+	decodeKeyFunc   func(key []byte) (e IndexItem, err error)
+	encodeValueFunc func(fields IndexItem) (value []byte, err error)
+	decodeValueFunc func(value []byte) (e IndexItem, err error)
+}
+
+// NewMockIndex returns a new MockIndex instance with defined name and
+// encoding functions. The name must be unique and will be validated
+// on database schema for a key prefix byte.
+// The data will not be saved on the DB itself, but using a provided
+// mock.NodeStore.
+func (db *DB) NewMockIndex(store *mock.NodeStore, name string, funcs IndexFuncs) (f MockIndex, err error) {
+	id, err := db.schemaIndexPrefix(name)
+	if err != nil {
+		return f, err
+	}
+	return MockIndex{
+		store:           store,
+		prefix:          []byte{id},
+		encodeKeyFunc:   newIndexEncodeKeyFunc(funcs.EncodeKey, id),
+		decodeKeyFunc:   newDecodeKeyFunc(funcs.DecodeKey),
+		encodeValueFunc: funcs.EncodeValue,
+		decodeValueFunc: funcs.DecodeValue,
+	}, nil
+}
+
+// Get accepts key fields represented as IndexItem to retrieve a
+// value from the index and return maximum available information
+// from the index represented as another IndexItem.
+func (f MockIndex) Get(keyFields IndexItem) (out IndexItem, err error) {
+	key, err := f.encodeKeyFunc(keyFields)
+	if err != nil {
+		return out, err
+	}
+	value, err := f.store.Get(key)
+	if err != nil {
+		return out, err
+	}
+	out, err = f.decodeValueFunc(value)
+	if err != nil {
+		return out, err
+	}
+	return out.Join(keyFields), nil
+}
+
+// Put accepts IndexItem to encode information from it
+// and save it to the database.
+func (f MockIndex) Put(i IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(i)
+	if err != nil {
+		return err
+	}
+	value, err := f.encodeValueFunc(i)
+	if err != nil {
+		return err
+	}
+	return f.store.Put(key, value)
+}
+
+// PutInBatch is the same as Put method.
+// Batch is ignored and the data is saved to the mock store instantly.
+func (f MockIndex) PutInBatch(_ *leveldb.Batch, i IndexItem) (err error) {
+	return f.Put(i)
+}
+
+// Delete accepts IndexItem to remove a key/value pair
+// form the database based on its fields.
+func (f MockIndex) Delete(keyFields IndexItem) (err error) {
+	key, err := f.encodeKeyFunc(keyFields)
+	if err != nil {
+		return err
+	}
+	return f.store.Delete(key)
+}
+
+// DeleteInBatch is the same as Delete just the operation.
+// Batch is ignored and the data is deleted on the mock store instantly.
+func (f MockIndex) DeleteInBatch(_ *leveldb.Batch, keyFields IndexItem) (err error) {
+	return f.Delete(keyFields)
+}

swarm/shed/index_mock_test.go

@@ -0,0 +1,45 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package shed
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+)
+
+// TestMockIndex validates put, get and delete functions of
+// the MockIndex implementation.
+func TestMockIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	globalStore := mem.NewGlobalStore()
+
+	index, err := db.NewMockIndex(
+		globalStore.NewNodeStore(common.HexToAddress("12345678")),
+		"retrieval",
+		retrievalIndexFuncs,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testIndex(t, db, index)
+}

swarm/shed/index_test.go

@@ -24,6 +24,9 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -49,7 +52,7 @@ var retrievalIndexFuncs = IndexFuncs{
 	},
 }
 
-// TestIndex validates put, get and delete functions of the index.
+// TestIndex validates put, get and delete functions of the Index implementation.
 func TestIndex(t *testing.T) {
 	db, cleanupFunc := newTestDB(t)
 	defer cleanupFunc()
@@ -59,6 +62,29 @@ func TestIndex(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	testIndex(t, db, index)
+}
+
+// TestIndex_NewMockIndex validates put, get and delete functions of
+// the MockIndex implementation, when constructed with Index.NewMockIndex function.
+func TestIndex_NewMockIndex(t *testing.T) {
+	db, cleanupFunc := newTestDB(t)
+	defer cleanupFunc()
+
+	index, err := db.NewIndex("retrieval", retrievalIndexFuncs)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	globalStore := mem.NewGlobalStore()
+
+	mockIndex := index.NewMockIndex(globalStore.NewNodeStore(common.HexToAddress("12345678")))
+
+	testIndex(t, db, mockIndex)
+}
+
+// testIndex validates put, get and delete functions of a index interface.
+func testIndex(t *testing.T, db *DB, index IndexInterface) {
 	t.Run("put", func(t *testing.T) {
 		want := IndexItem{
 			Address:        []byte("put-hash"),
@@ -66,7 +92,7 @@ func TestIndex(t *testing.T) {
 			StoreTimestamp: time.Now().UTC().UnixNano(),
 		}
 
-		err = index.Put(want)
+		err := index.Put(want)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -108,7 +134,7 @@ func TestIndex(t *testing.T) {
 
 		batch := new(leveldb.Batch)
 		index.PutInBatch(batch, want)
-		db.WriteBatch(batch)
+		err := db.WriteBatch(batch)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -150,7 +176,7 @@ func TestIndex(t *testing.T) {
 			StoreTimestamp: time.Now().UTC().UnixNano(),
 		}
 
-		err = index.Put(want)
+		err := index.Put(want)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -169,11 +195,15 @@ func TestIndex(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		wantErr := leveldb.ErrNotFound
+		if _, ok := index.(MockIndex); ok {
+			wantErr = mock.ErrNotFound
+		}
 		got, err = index.Get(IndexItem{
 			Address: want.Address,
 		})
-		if err != leveldb.ErrNotFound {
+		if err != wantErr {
-			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+			t.Fatalf("got error %v, want %v", err, wantErr)
 		}
 	})
 
@@ -184,7 +214,7 @@ func TestIndex(t *testing.T) {
 			StoreTimestamp: time.Now().UTC().UnixNano(),
 		}
 
-		err = index.Put(want)
+		err := index.Put(want)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -205,11 +235,15 @@ func TestIndex(t *testing.T) {
 			t.Fatal(err)
 		}
 
+		wantErr := leveldb.ErrNotFound
+		if _, ok := index.(MockIndex); ok {
+			wantErr = mock.ErrNotFound
+		}
 		got, err = index.Get(IndexItem{
 			Address: want.Address,
 		})
-		if err != leveldb.ErrNotFound {
+		if err != wantErr {
-			t.Fatalf("got error %v, want %v", err, leveldb.ErrNotFound)
+			t.Fatalf("got error %v, want %v", err, wantErr)
 		}
 	})
 }

swarm/storage/mock/db/db.go

@@ -86,6 +86,13 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
 	return s.db.Write(batch, nil)
 }
 
+// Delete removes the chunk reference to node with address addr.
+func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
+	batch := new(leveldb.Batch)
+	batch.Delete(nodeDBKey(addr, key))
+	return s.db.Write(batch, nil)
+}
+
 // HasKey returns whether a node with addr contains the key.
 func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
 	has, err := s.db.Has(nodeDBKey(addr, key), nil)

swarm/storage/mock/mem/mem.go

@@ -83,6 +83,22 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
 	return nil
 }
 
+// Delete removes the chunk data for node with address addr.
+func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var count int
+	if _, ok := s.nodes[string(key)]; ok {
+		delete(s.nodes[string(key)], addr)
+		count = len(s.nodes[string(key)])
+	}
+	if count == 0 {
+		delete(s.data, string(key))
+	}
+	return nil
+}
+
 // HasKey returns whether a node with addr contains the key.
 func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
 	s.mu.Lock()

swarm/storage/mock/mock.go

@@ -70,6 +70,12 @@ func (n *NodeStore) Put(key []byte, data []byte) error {
 	return n.store.Put(n.addr, key, data)
 }
 
+// Delete removes chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Delete(key []byte) error {
+	return n.store.Delete(n.addr, key)
+}
+
 // GlobalStorer defines methods for mock db store
 // that stores chunk data for all swarm nodes.
 // It is used in tests to construct mock NodeStores
@@ -77,6 +83,7 @@ func (n *NodeStore) Put(key []byte, data []byte) error {
 type GlobalStorer interface {
 	Get(addr common.Address, key []byte) (data []byte, err error)
 	Put(addr common.Address, key []byte, data []byte) error
+	Delete(addr common.Address, key []byte) error
 	HasKey(addr common.Address, key []byte) bool
 	// NewNodeStore creates an instance of NodeStore
 	// to be used by a single swarm node with

swarm/storage/mock/rpc/rpc.go

@@ -73,6 +73,12 @@ func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
 	return err
 }
 
+// Delete calls a Delete method to RPC server.
+func (s *GlobalStore) Delete(addr common.Address, key []byte) error {
+	err := s.client.Call(nil, "mockStore_delete", addr, key)
+	return err
+}
+
 // HasKey calls a HasKey method to RPC server.
 func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
 	var has bool

swarm/storage/mock/test/test.go

@@ -72,6 +72,31 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
 				}
 			}
 		}
+		t.Run("delete", func(t *testing.T) {
+			chunkAddr := storage.Address([]byte("1234567890abcd"))
+			for _, addr := range addrs {
+				err := globalStore.Put(addr, chunkAddr, []byte("data"))
+				if err != nil {
+					t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
+				}
+			}
+			firstNodeAddr := addrs[0]
+			if err := globalStore.Delete(firstNodeAddr, chunkAddr); err != nil {
+				t.Fatalf("delete from store %s key %s: %v", firstNodeAddr.Hex(), chunkAddr.Hex(), err)
+			}
+			for i, addr := range addrs {
+				_, err := globalStore.Get(addr, chunkAddr)
+				if i == 0 {
+					if err != mock.ErrNotFound {
+						t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err)
+					}
+				} else {
+					if err != nil {
+						t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
+					}
+				}
+			}
+		})
 	})
 
 	t.Run("NodeStore", func(t *testing.T) {
@@ -114,6 +139,34 @@ func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
 				}
 			}
 		}
+		t.Run("delete", func(t *testing.T) {
+			chunkAddr := storage.Address([]byte("1234567890abcd"))
+			var chosenStore *mock.NodeStore
+			for addr, store := range nodes {
+				if chosenStore == nil {
+					chosenStore = store
+				}
+				err := store.Put(chunkAddr, []byte("data"))
+				if err != nil {
+					t.Fatalf("put data to store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
+				}
+			}
+			if err := chosenStore.Delete(chunkAddr); err != nil {
+				t.Fatalf("delete key %s: %v", chunkAddr.Hex(), err)
+			}
+			for addr, store := range nodes {
+				_, err := store.Get(chunkAddr)
+				if store == chosenStore {
+					if err != mock.ErrNotFound {
+						t.Errorf("get data from store %s key %s: expected mock.ErrNotFound error, got %v", addr.Hex(), chunkAddr.Hex(), err)
+					}
+				} else {
+					if err != nil {
+						t.Errorf("get data from store %s key %s: %v", addr.Hex(), chunkAddr.Hex(), err)
+					}
+				}
+			}
+		})
 	})
 }