Merge pull request #198 from ethersphere/mock-store

cmd/swarm, swarm: implement mock datastore (#161)

cmd/swarm/main.go

@@ -534,7 +534,8 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 			}
 		}
 
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled)
+		// In production, mockStore must be always nil.
+		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

swarm/storage/dbstore.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 )
@@ -75,6 +76,15 @@ type DbStore struct {
 	hashfunc SwarmHasher
 
 	lock sync.Mutex
+
+	// Functions encodeDataFunc is used to bypass
+	// the default functionality of DbStore with
+	// mock.NodeStore for testing purposes.
+	encodeDataFunc func(chunk *Chunk) []byte
+	// If getDataFunc is defined, it will be used for
+	// retrieving the chunk data instead from the local
+	// LevelDB database.
+	getDataFunc func(key Key) (data []byte, err error)
 }
 
 func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) {
@@ -82,6 +92,9 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *
 
 	s.hashfunc = hash
 
+	// associate encodeData with default functionality
+	s.encodeDataFunc = encodeData
+
 	s.db, err = NewLDBDatabase(path)
 	if err != nil {
 		return
@@ -106,6 +119,22 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *
 	return
 }
 
+// NewMockDbStore creates a new instance of DbStore with
+// mockStore set to a provided value. If mockStore argument is nil,
+// this function behaves exactly as NewDbStore.
+func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, radius int, mockStore *mock.NodeStore) (s *DbStore, err error) {
+	s, err = NewDbStore(path, hash, capacity, radius)
+	if err != nil {
+		return nil, err
+	}
+	// replace put and get with mock store functionality
+	if mockStore != nil {
+		s.encodeDataFunc = newMockEncodeDataFunc(mockStore)
+		s.getDataFunc = newMockGetDataFunc(mockStore)
+	}
+	return
+}
+
 type dpaDBIndex struct {
 	Idx    uint64
 	Access uint64
@@ -409,7 +438,7 @@ func (s *DbStore) Put(chunk *Chunk) {
 		return // already exists, only update access
 	}
 
-	data := encodeData(chunk)
+	data := s.encodeDataFunc(chunk)
 	//data := ethutil.Encode([]interface{}{entry})
 
 	if s.entryCnt >= s.capacity {
@@ -440,6 +469,19 @@ func (s *DbStore) Put(chunk *Chunk) {
 	log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx))
 }
 
+// newMockEncodeDataFunc returns a function that stores the chunk data
+// to a mock store to bypass the default functionality encodeData.
+// The constructed function always returns the nil data, as DbStore does
+// not need to store the data, but still need to create the index.
+func newMockEncodeDataFunc(mockStore *mock.NodeStore) func(chunk *Chunk) []byte {
+	return func(chunk *Chunk) []byte {
+		if err := mockStore.Put(chunk.Key, chunk.SData); err != nil {
+			log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, chunk.Key.Log(), err))
+		}
+		return nil
+	}
+}
+
 // try to find index; if found, update access cnt and return true
 func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
 	idata, err := s.db.Get(ikey)
@@ -469,11 +511,20 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
 
 	if s.tryAccessIdx(getIndexKey(key), &index) {
 		var data []byte
-		data, err = s.db.Get(getDataKey(index.Idx))
+		if s.getDataFunc != nil {
-		if err != nil {
+			// if getDataFunc is defined, use it to retrieve the chunk data
-			log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
+			data, err = s.getDataFunc(key)
-			s.delete(index.Idx, getIndexKey(key))
+			if err != nil {
-			return
+				return
+			}
+		} else {
+			// default DbStore functionality to retrieve chunk data
+			data, err = s.db.Get(getDataKey(index.Idx))
+			if err != nil {
+				log.Trace(fmt.Sprintf("DBStore: Chunk %v found but could not be accessed: %v", key.Log(), err))
+				s.delete(index.Idx, getIndexKey(key))
+				return
+			}
 		}
 
 		if s.hashfunc != nil {
@@ -490,12 +541,26 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
 			Key: key,
 		}
 		decodeData(data, chunk)
+
 	} else {
 		err = notFound
 	}
 
 	return
+}
 
+// newMockGetFunc returns a function that reads chunk data from
+// the mock database, which is used as the value for DbStore.getFunc
+// to bypass the default functionality of DbStore with a mock store.
+func newMockGetDataFunc(mockStore *mock.NodeStore) func(key Key) (data []byte, err error) {
+	return func(key Key) (data []byte, err error) {
+		data, err = mockStore.Get(key)
+		if err == mock.ErrNotFound {
+			// preserve notFound error
+			err = notFound
+		}
+		return data, err
+	}
 }
 
 func (s *DbStore) updateAccessCnt(key Key) {

swarm/storage/dbstore_test.go

@@ -19,9 +19,12 @@ package storage
 import (
 	"bytes"
 	"io/ioutil"
+	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 )
 
 func initDbStore(t *testing.T) *DbStore {
@@ -189,3 +192,76 @@ func TestDbStoreSyncIterator(t *testing.T) {
 		t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
 	}
 }
+
+func initMockDbStore(t *testing.T, mockStore *mock.NodeStore) *DbStore {
+	dir, err := ioutil.TempDir("", "bzz-storage-test-mock")
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius, mockStore)
+	if err != nil {
+		t.Fatal("can't create store:", err)
+	}
+	return m
+}
+
+// testMockDbStore runs the same tests as testDbStore but with mock store configured.
+// It also verifies if mock global store is storing the chunk data.
+func testMockDbStore(l int64, branches int64, t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
+	mockStore := globalStore.NewNodeStore(addr)
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+
+	key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7"))
+	data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096)
+
+	m.Put(&Chunk{
+		Key:   key,
+		SData: data,
+	})
+
+	_, err := globalStore.Get(addr, key)
+	if err != nil {
+		t.Errorf("unexpected error getting the data from global mock store: %v", err)
+	}
+
+	if !globalStore.HasKey(addr, key) {
+		t.Error("key not found in global store")
+	}
+
+	testStore(m, l, branches, t)
+
+}
+
+func TestMockDbStore128_0x1000000(t *testing.T) {
+	testMockDbStore(0x1000000, 128, t)
+}
+
+func TestMockDbStore128_10000_(t *testing.T) {
+	testMockDbStore(10000, 128, t)
+}
+
+func TestMockDbStore128_1000_(t *testing.T) {
+	testMockDbStore(1000, 128, t)
+}
+
+func TestMockDbStore128_100_(t *testing.T) {
+	testMockDbStore(100, 128, t)
+}
+
+func TestMockDbStore2_100_(t *testing.T) {
+	testMockDbStore(100, 2, t)
+}
+
+func TestMockDbStoreNotFound(t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed"))
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+	_, err := m.Get(ZeroKey)
+	if err != notFound {
+		t.Errorf("Expected notFound, got %v", err)
+	}
+}

swarm/storage/localstore.go

@@ -18,6 +18,8 @@ package storage
 
 import (
 	"encoding/binary"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 )
 
 // LocalStore is a combination of inmemory db over a disk persisted db
@@ -27,9 +29,10 @@ type LocalStore struct {
 	DbStore  ChunkStore
 }
 
-// This constructor uses MemStore and DbStore as components
+// This constructor uses MemStore and DbStore as components.
-func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) {
+// If mockStore is not nil, it will be used by DbStore to store chunk data.
-	dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius)
+func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore *mock.NodeStore) (*LocalStore, error) {
+	dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius, mockStore)
 	if err != nil {
 		return nil, err
 	}

swarm/storage/mock/db/db.go

@@ -0,0 +1,236 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package db implements a mock store that keeps all chunk data in LevelDB database.
+package db
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore contains the LevelDB database that is storing
+// chunk data for all swarm nodes.
+// Closing the GlobalStore with Close method is required to
+// release resources used by the database.
+type GlobalStore struct {
+	db *leveldb.DB
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(path string) (s *GlobalStore, err error) {
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &GlobalStore{
+		db: db,
+	}, nil
+}
+
+// Close releases the resources used by the underlying LevelDB.
+func (s *GlobalStore) Close() error {
+	return s.db.Close()
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		return nil, mock.ErrNotFound
+	}
+	if !has {
+		return nil, mock.ErrNotFound
+	}
+	data, err = s.db.Get(dataDBKey(key), nil)
+	if err == leveldb.ErrNotFound {
+		err = mock.ErrNotFound
+	}
+	return
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	batch := new(leveldb.Batch)
+	batch.Put(nodeDBKey(addr, key), nil)
+	batch.Put(dataDBKey(key), data)
+	return s.db.Write(batch, nil)
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		has = false
+	}
+	return has
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		batch := new(leveldb.Batch)
+		for _, addr := range c.Addrs {
+			batch.Put(nodeDBKeyHex(addr, hdr.Name), nil)
+		}
+
+		batch.Put(dataDBKey(common.Hex2Bytes(hdr.Name)), c.Data)
+		if err = s.db.Write(batch, nil); err != nil {
+			return n, err
+		}
+
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number fo chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+
+	iter := s.db.NewIterator(util.BytesPrefix(nodeKeyPrefix), nil)
+	defer iter.Release()
+
+	var currentKey string
+	var addrs []common.Address
+
+	saveChunk := func(hexKey string) error {
+		key := common.Hex2Bytes(hexKey)
+
+		data, err := s.db.Get(dataDBKey(key), nil)
+		if err != nil {
+			return err
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: addrs,
+			Data:  data,
+		}); err != nil {
+			return err
+		}
+
+		d := buf.Bytes()
+		hdr := &tar.Header{
+			Name: hexKey,
+			Mode: 0644,
+			Size: int64(len(d)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := tw.Write(d); err != nil {
+			return err
+		}
+		n++
+		return nil
+	}
+
+	for iter.Next() {
+		k := bytes.TrimPrefix(iter.Key(), nodeKeyPrefix)
+		i := bytes.Index(k, []byte("-"))
+		if i < 0 {
+			continue
+		}
+		hexKey := string(k[:i])
+
+		if currentKey == "" {
+			currentKey = hexKey
+		}
+
+		if hexKey != currentKey {
+			if err = saveChunk(currentKey); err != nil {
+				return n, err
+			}
+
+			addrs = addrs[:0]
+		}
+
+		currentKey = hexKey
+		addrs = append(addrs, common.BytesToAddress(k[i:]))
+	}
+
+	if len(addrs) > 0 {
+		if err = saveChunk(currentKey); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+var (
+	nodeKeyPrefix = []byte("node-")
+	dataKeyPrefix = []byte("data-")
+)
+
+// nodeDBKey constructs a database key for key/node mappings.
+func nodeDBKey(addr common.Address, key []byte) []byte {
+	return nodeDBKeyHex(addr, common.Bytes2Hex(key))
+}
+
+// nodeDBKeyHex constructs a database key for key/node mappings
+// using the hexadecimal string representation of the key.
+func nodeDBKeyHex(addr common.Address, hexKey string) []byte {
+	return append(append(nodeKeyPrefix, []byte(hexKey+"-")...), addr[:]...)
+}
+
+// dataDBkey constructs a database key for key/data storage.
+func dataDBKey(key []byte) []byte {
+	return append(dataKeyPrefix, key...)
+}

swarm/storage/mock/db/db_test.go

@@ -0,0 +1,73 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+// TestDBStore is running a test.MockStore tests
+// using test.MockStore function.
+func TestDBStore(t *testing.T) {
+	dir, err := ioutil.TempDir("", "mock_"+t.Name())
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir)
+
+	store, err := NewGlobalStore(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+
+	test.MockStore(t, store, 100)
+}
+
+// TestImportExport is running a test.ImportExport tests
+// using test.MockStore function.
+func TestImportExport(t *testing.T) {
+	dir1, err := ioutil.TempDir("", "mock_"+t.Name()+"_exporter")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir1)
+
+	store1, err := NewGlobalStore(dir1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store1.Close()
+
+	dir2, err := ioutil.TempDir("", "mock_"+t.Name()+"_importer")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir2)
+
+	store2, err := NewGlobalStore(dir2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store2.Close()
+
+	test.ImportExport(t, store1, store2, 100)
+}

swarm/storage/mock/mem/mem.go

@@ -0,0 +1,175 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mem implements a mock store that keeps all chunk data in memory.
+// While it can be used for testing on smaller scales, the main purpose of this
+// package is to provide the simplest reference implementation of a mock store.
+package mem
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore stores all chunk data and also keys and node addresses relations.
+// It implements mock.GlobalStore interface.
+type GlobalStore struct {
+	nodes map[string]map[common.Address]struct{}
+	data  map[string][]byte
+	mu    sync.Mutex
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore() *GlobalStore {
+	return &GlobalStore{
+		nodes: make(map[string]map[common.Address]struct{}),
+		data:  make(map[string][]byte),
+	}
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)][addr]; !ok {
+		return nil, mock.ErrNotFound
+	}
+
+	data, ok := s.data[string(key)]
+	if !ok {
+		return nil, mock.ErrNotFound
+	}
+	return data, nil
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)]; !ok {
+		s.nodes[string(key)] = make(map[common.Address]struct{})
+	}
+	s.nodes[string(key)][addr] = struct{}{}
+	s.data[string(key)] = data
+	return nil
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	_, ok := s.nodes[string(key)][addr]
+	return ok
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		addrs := make(map[common.Address]struct{})
+		for _, a := range c.Addrs {
+			addrs[a] = struct{}{}
+		}
+
+		key := string(common.Hex2Bytes(hdr.Name))
+		s.nodes[key] = addrs
+		s.data[key] = c.Data
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number of chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+	for key, addrs := range s.nodes {
+		al := make([]common.Address, 0, len(addrs))
+		for a := range addrs {
+			al = append(al, a)
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: al,
+			Data:  s.data[key],
+		}); err != nil {
+			return n, err
+		}
+
+		data := buf.Bytes()
+		hdr := &tar.Header{
+			Name: common.Bytes2Hex([]byte(key)),
+			Mode: 0644,
+			Size: int64(len(data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return n, err
+		}
+		if _, err := tw.Write(data); err != nil {
+			return n, err
+		}
+		n++
+	}
+	return n, err
+}

swarm/storage/mock/mem/mem_test.go

@@ -0,0 +1,36 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mem
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+// TestDBStore is running test for a GlobalStore
+// using test.MockStore function.
+func TestMemStore(t *testing.T) {
+	test.MockStore(t, NewGlobalStore(), 100)
+}
+
+// TestImportExport is running tests for importing and
+// exporting data between two GlobalStores
+// using test.ImportExport function.
+func TestImportExport(t *testing.T) {
+	test.ImportExport(t, NewGlobalStore(), NewGlobalStore(), 100)
+}

swarm/storage/mock/mock.go

@@ -0,0 +1,111 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mock defines types that are used by different implementations
+// of mock storages.
+//
+// Implementations of mock storages are located in directories
+// under this package:
+//
+//  - db - LevelDB backend
+//  - mem - in memory map backend
+//  - rpc - RPC client that can connect to other backends
+//
+// Mock storages can implement Importer and Exporter interfaces
+// for importing and exporting all chunk data that they contain.
+// The exported file is a tar archive with all files named by
+// hexadecimal representations of chunk keys and with content
+// with JSON-encoded ExportedChunk structure. Exported format
+// should be preserved across all mock store implementations.
+package mock
+
+import (
+	"errors"
+	"io"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// ErrNotFound indicates that the chunk is not found.
+var ErrNotFound = errors.New("not found")
+
+// NodeStore holds the node address and a reference to the GlobalStore
+// in order to access and store chunk data only for one node.
+type NodeStore struct {
+	store GlobalStorer
+	addr  common.Address
+}
+
+// NewNodeStore creates a new instance of NodeStore that keeps
+// chunk data using GlobalStorer with a provided address.
+func NewNodeStore(addr common.Address, store GlobalStorer) *NodeStore {
+	return &NodeStore{
+		store: store,
+		addr:  addr,
+	}
+}
+
+// Get returns chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Get(key []byte) (data []byte, err error) {
+	return n.store.Get(n.addr, key)
+}
+
+// Put saves chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Put(key []byte, data []byte) error {
+	return n.store.Put(n.addr, key, data)
+}
+
+// GlobalStorer defines methods for mock db store
+// that stores chunk data for all swarm nodes.
+// It is used in tests to construct mock NodeStores
+// for swarm nodes and to track and validate chunks.
+type GlobalStorer interface {
+	Get(addr common.Address, key []byte) (data []byte, err error)
+	Put(addr common.Address, key []byte, data []byte) error
+	HasKey(addr common.Address, key []byte) bool
+	// NewNodeStore creates an instance of NodeStore
+	// to be used by a single swarm node with
+	// address addr.
+	NewNodeStore(addr common.Address) *NodeStore
+}
+
+// Importer defines method for importing mock store data
+// from an exported tar archive.
+type Importer interface {
+	Import(r io.Reader) (n int, err error)
+}
+
+// Exporter defines method for exporting mock store data
+// to a tar archive.
+type Exporter interface {
+	Export(w io.Writer) (n int, err error)
+}
+
+// ImportExporter is an interface for importing and exporting
+// mock store data to and from a tar archive.
+type ImportExporter interface {
+	Importer
+	Exporter
+}
+
+// ExportedChunk is the structure that is saved in tar archive for
+// each chunk as JSON-encoded bytes.
+type ExportedChunk struct {
+	Data  []byte           `json:"d"`
+	Addrs []common.Address `json:"a"`
+}

swarm/storage/mock/rpc/rpc.go

@@ -0,0 +1,84 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package rpc implements an RPC client that connect to a centralized mock store.
+// Centralazied mock store can be any other mock store implementation that is
+// registered to Ethereum RPC server under mockStore name. Methods that defines
+// mock.GlobalStore are the same that are used by RPC. Example:
+//
+//   server := rpc.NewServer()
+//   server.RegisterName("mockStore", mem.NewGlobalStore())
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore is rpc.Client that connects to a centralized mock store.
+// Closing GlobalStore instance is required to release RPC client resources.
+type GlobalStore struct {
+	client *rpc.Client
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(client *rpc.Client) *GlobalStore {
+	return &GlobalStore{
+		client: client,
+	}
+}
+
+// Close closes RPC client.
+func (s *GlobalStore) Close() error {
+	s.client.Close()
+	return nil
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) *mock.NodeStore {
+	return mock.NewNodeStore(addr, s)
+}
+
+// Get calls a Get method to RPC server.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	err = s.client.Call(&data, "mockStore_get", addr, key)
+	if err != nil && err.Error() == "not found" {
+		// pass the mock package value of error instead an rpc error
+		return data, mock.ErrNotFound
+	}
+	return data, err
+}
+
+// Put calls a Put method to RPC server.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	err := s.client.Call(nil, "mockStore_put", addr, key, data)
+	return err
+}
+
+// HasKey calls a HasKey method to RPC server.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	var has bool
+	if err := s.client.Call(&has, "mockStore_hasKey", addr, key); err != nil {
+		log.Error(fmt.Sprintf("mock store HasKey: addr %s, key %064x: %v", addr, key, err))
+		return false
+	}
+	return has
+}

swarm/storage/mock/rpc/rpc_test.go

@@ -0,0 +1,41 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+// TestDBStore is running test for a GlobalStore
+// using test.MockStore function.
+func TestRPCStore(t *testing.T) {
+	serverStore := mem.NewGlobalStore()
+
+	server := rpc.NewServer()
+	if err := server.RegisterName("mockStore", serverStore); err != nil {
+		t.Fatal(err)
+	}
+
+	store := NewGlobalStore(rpc.DialInProc(server))
+	defer store.Close()
+
+	test.MockStore(t, store, 100)
+}

swarm/storage/mock/test/test.go

@@ -0,0 +1,186 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package test provides functions that are used for testing
+// GlobalStorer implementations.
+package test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// MockStore creates NodeStore instances from provided GlobalStorer,
+// each one with a unique address, stores different chunks on them
+// and checks if they are retrievable or not on all nodes.
+// Attribute n defines the number of NodeStores that will be created.
+func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
+	t.Run("GlobalStore", func(t *testing.T) {
+		addrs := make([]common.Address, n)
+		for i := 0; i < n; i++ {
+			addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+		}
+
+		for i, addr := range addrs {
+			key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			globalStore.Put(addr, key, data)
+
+			for _, cAddr := range addrs {
+				cData, err := globalStore.Get(cAddr, key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+
+	t.Run("NodeStore", func(t *testing.T) {
+		nodes := make(map[common.Address]*mock.NodeStore)
+		for i := 0; i < n; i++ {
+			addr := common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+			nodes[addr] = globalStore.NewNodeStore(addr)
+		}
+
+		i := 0
+		for addr, store := range nodes {
+			i++
+			key := storage.Key(append(addr[:], []byte(fmt.Sprintf("%x", i))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			store.Put(key, data)
+
+			for cAddr, cStore := range nodes {
+				cData, err := cStore.Get(key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+}
+
+// ImportExport saves chunks to the outStore, exports them to the tar archive,
+// imports tar archive to the inStore and checks if all chunks are imported correctly.
+func ImportExport(t *testing.T, outStore, inStore mock.GlobalStorer, n int) {
+	exporter, ok := outStore.(mock.Exporter)
+	if !ok {
+		t.Fatal("outStore does not implement mock.Exporter")
+	}
+	importer, ok := inStore.(mock.Importer)
+	if !ok {
+		t.Fatal("inStore does not implement mock.Importer")
+	}
+	addrs := make([]common.Address, n)
+	for i := 0; i < n; i++ {
+		addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		outStore.Put(addr, key, data)
+	}
+
+	r, w := io.Pipe()
+	defer r.Close()
+
+	go func() {
+		defer w.Close()
+		if _, err := exporter.Export(w); err != nil {
+			t.Fatalf("export: %v", err)
+		}
+	}()
+
+	if _, err := importer.Import(r); err != nil {
+		t.Fatalf("import: %v", err)
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		for _, cAddr := range addrs {
+			cData, err := inStore.Get(cAddr, key)
+			if cAddr == addr {
+				if err != nil {
+					t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+				}
+				if !bytes.Equal(data, cData) {
+					t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+				}
+				if !inStore.HasKey(cAddr, key) {
+					t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+				}
+			} else {
+				if err != mock.ErrNotFound {
+					t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+				}
+				if len(cData) > 0 {
+					t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+				}
+				if inStore.HasKey(cAddr, key) {
+					t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+				}
+			}
+		}
+	}
+}

swarm/swarm.go

@@ -41,6 +41,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 )
 
 // the swarm stack
@@ -79,7 +80,9 @@ func (self *Swarm) API() *SwarmAPI {
 
 // creates a new swarm service instance
 // implements node.Service
-func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool) (self *Swarm, err error) {
+// If mockStore is not nil, it will be used as the storage for chunk data.
+// MockStore should be used only for testing.
+func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, mockStore *mock.NodeStore) (self *Swarm, err error) {
 	if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) {
 		return nil, fmt.Errorf("empty public key")
 	}
@@ -97,7 +100,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
 	log.Debug(fmt.Sprintf("Setting up Swarm service components"))
 
 	hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
-	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams)
+	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, mockStore)
 	if err != nil {
 		return
 	}