cmd/swarm, swarm: implement mock datastore (#161)

Packages under swarm/storage/mock define and implement mock storages that
keep chunk data of all swarm nodes in a centralized way for testing
purposes. They provide:
  - create individual swarm node mock stores that can be injected to
    swarm to bypass the DbStore chunk data storing
  - methods to inspect on which nodes are chunk stored
  - import/export functionality

Three implementations of mock stores are:
  - swarm/storage/mem - a memory backed storage
  - swarm/storage/db - a LevelDB backed storage
  - swarm/storage/rpc - a storage that connects to other storages by RPC

Bypassing is done only for the chunk data, not for the local chunk indexes
that DbStore creates which can not be shared between swarm nodes.

cmd/swarm/main.go

@@ -534,7 +534,8 @@ func registerBzzService(bzzconfig *bzzapi.Config, ctx *cli.Context, stack *node.
 			}
 		}
 
-		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled)
+		// In production, mockStore must be always nil.
+		return swarm.NewSwarm(ctx, swapClient, ensClient, bzzconfig, bzzconfig.SwapEnabled, bzzconfig.SyncEnabled, bzzconfig.Cors, bzzconfig.PssEnabled, nil)
 	}
 	//register within the ethereum node
 	if err := stack.Register(boot); err != nil {

swarm/storage/dbstore.go

@@ -34,6 +34,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 )
@@ -75,6 +76,14 @@ type DbStore struct {
 	hashfunc SwarmHasher
 
 	lock sync.Mutex
+
+	// Functions putDataFunc and getFunc are used for
+	// saving and retreiving chunk data from the store.
+	// They must be set on DbStore initialization and must not be nil.
+	// They are used to bypass the default functionality of DbStore with
+	// mock.NodeStorer for testing purposes.
+	putDataFunc func(batch *leveldb.Batch, _ Key, data []byte)
+	getFunc     func(key Key) (chunk *Chunk, err error)
 }
 
 func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *DbStore, err error) {
@@ -82,6 +91,10 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *
 
 	s.hashfunc = hash
 
+	// associate put and get with default functionality
+	s.putDataFunc = s.dbPutDataFunc
+	s.getFunc = s.dbGetFunc
+
 	s.db, err = NewLDBDatabase(path)
 	if err != nil {
 		return
@@ -106,6 +119,22 @@ func NewDbStore(path string, hash SwarmHasher, capacity uint64, radius int) (s *
 	return
 }
 
+// NewMockDbStore creates a new instance of DbStore with
+// mockStore set to a provided value. If mockStore argument is nil,
+// this function behaves exactly as NewDbStore.
+func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, radius int, mockStore mock.NodeStorer) (s *DbStore, err error) {
+	s, err = NewDbStore(path, hash, capacity, radius)
+	if err != nil {
+		return nil, err
+	}
+	// replace put and get with mock store functionality
+	if mockStore != nil {
+		s.putDataFunc = newMockPutDataFunc(mockStore)
+		s.getFunc = newMockGetFunc(mockStore)
+	}
+	return
+}
+
 type dpaDBIndex struct {
 	Idx    uint64
 	Access uint64
@@ -418,7 +447,7 @@ func (s *DbStore) Put(chunk *Chunk) {
 
 	batch := new(leveldb.Batch)
 
-	batch.Put(getDataKey(s.dataIdx), data)
+	s.putDataFunc(batch, chunk.Key, data)
 
 	index.Idx = s.dataIdx
 	s.updateIndexAccess(&index)
@@ -440,6 +469,20 @@ func (s *DbStore) Put(chunk *Chunk) {
 	log.Trace(fmt.Sprintf("DbStore.Put: %v. db storage counter: %v ", chunk.Key.Log(), s.dataIdx))
 }
 
+func (s *DbStore) dbPutDataFunc(batch *leveldb.Batch, _ Key, data []byte) {
+	batch.Put(getDataKey(s.dataIdx), data)
+}
+
+// newMockPutDataFunc returns a function that stores the chunk data
+// to a mock store to bypass the default functionality of DbStore.
+func newMockPutDataFunc(mockStore mock.NodeStorer) func(_ *leveldb.Batch, key Key, data []byte) {
+	return func(_ *leveldb.Batch, key Key, data []byte) {
+		if err := mockStore.Put(key, data); err != nil {
+			log.Error(fmt.Sprintf("%T: Chunk %v put: %v", mockStore, key.Log(), err))
+		}
+	}
+}
+
 // try to find index; if found, update access cnt and return true
 func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
 	idata, err := s.db.Get(ikey)
@@ -462,6 +505,12 @@ func (s *DbStore) tryAccessIdx(ikey []byte, index *dpaDBIndex) bool {
 }
 
 func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
+	return s.getFunc(key)
+}
+
+// dbGetFunc provides the default functionality for accessing
+// the chunk data from LevelDB.
+func (s *DbStore) dbGetFunc(key Key) (chunk *Chunk, err error) {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 
@@ -498,6 +547,27 @@ func (s *DbStore) Get(key Key) (chunk *Chunk, err error) {
 
 }
 
+// newMockGetFunc returns a function that reads chunk data from
+// the mock database, which is used as the value for DbStore.getFunc
+// to bypass the default functionality of DbStore with a mock store.
+func newMockGetFunc(mockStore mock.NodeStorer) func(key Key) (chunk *Chunk, err error) {
+	return func(key Key) (chunk *Chunk, err error) {
+		data, err := mockStore.Get(key)
+		if err != nil {
+			if err == mock.ErrNotFound {
+				// preserve notFound error
+				err = notFound
+			}
+			return nil, err
+		}
+		chunk = &Chunk{
+			Key: key,
+		}
+		decodeData(data, chunk)
+		return chunk, nil
+	}
+}
+
 func (s *DbStore) updateAccessCnt(key Key) {
 
 	s.lock.Lock()

swarm/storage/dbstore_test.go

@@ -19,9 +19,12 @@ package storage
 import (
 	"bytes"
 	"io/ioutil"
+	"strings"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
 )
 
 func initDbStore(t *testing.T) *DbStore {
@@ -189,3 +192,74 @@ func TestDbStoreSyncIterator(t *testing.T) {
 		t.Fatalf("Expected %v chunk, got %v", keys[3], res[0])
 	}
 }
+
+func initMockDbStore(t *testing.T, mockStore mock.NodeStorer) *DbStore {
+	dir, err := ioutil.TempDir("", "bzz-storage-test-mock")
+	if err != nil {
+		t.Fatal(err)
+	}
+	m, err := NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, defaultRadius, mockStore)
+	if err != nil {
+		t.Fatal("can't create store:", err)
+	}
+	return m
+}
+
+func testMockDbStore(l int64, branches int64, t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
+	mockStore := globalStore.NewNodeStore(addr)
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+
+	key := Key(common.Hex2Bytes("fed1911825fc6a02ebfd19ab218a20455d8d7d275f8bf4d8244eb04364fae6f7"))
+	data := common.Hex2BytesFixed(strings.Repeat("1234567890abcdf", 10), 4096)
+
+	m.Put(&Chunk{
+		Key:   key,
+		SData: data,
+	})
+
+	_, err := globalStore.Get(addr, key)
+	if err != nil {
+		t.Errorf("unexpected error getting the data from global mock store: %v", err)
+	}
+
+	if !globalStore.HasKey(addr, key) {
+		t.Error("key not found in global store")
+	}
+
+	testStore(m, l, branches, t)
+
+}
+
+func TestMockDbStore128_0x1000000(t *testing.T) {
+	testMockDbStore(0x1000000, 128, t)
+}
+
+func TestMockDbStore128_10000_(t *testing.T) {
+	testMockDbStore(10000, 128, t)
+}
+
+func TestMockDbStore128_1000_(t *testing.T) {
+	testMockDbStore(1000, 128, t)
+}
+
+func TestMockDbStore128_100_(t *testing.T) {
+	testMockDbStore(100, 128, t)
+}
+
+func TestMockDbStore2_100_(t *testing.T) {
+	testMockDbStore(100, 2, t)
+}
+
+func TestMockDbStoreNotFound(t *testing.T) {
+	globalStore := mem.NewGlobalStore()
+	mockStore := globalStore.NewNodeStore(common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed"))
+	m := initMockDbStore(t, mockStore)
+	defer m.Close()
+	_, err := m.Get(ZeroKey)
+	if err != notFound {
+		t.Errorf("Expected notFound, got %v", err)
+	}
+}

swarm/storage/localstore.go

@@ -18,6 +18,8 @@ package storage
 
 import (
 	"encoding/binary"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 )
 
 // LocalStore is a combination of inmemory db over a disk persisted db
@@ -27,9 +29,10 @@ type LocalStore struct {
 	DbStore  ChunkStore
 }
 
-// This constructor uses MemStore and DbStore as components
+// This constructor uses MemStore and DbStore as components.
-func NewLocalStore(hash SwarmHasher, params *StoreParams) (*LocalStore, error) {
+// If mockStore is not nil, it will be used by DbStore to store chunk data.
-	dbStore, err := NewDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius)
+func NewLocalStore(hash SwarmHasher, params *StoreParams, mockStore mock.NodeStorer) (*LocalStore, error) {
+	dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, params.Radius, mockStore)
 	if err != nil {
 		return nil, err
 	}

swarm/storage/mock/db/db.go

@@ -0,0 +1,258 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package db implements a mock store that keeps all chunk data in LevelDB database.
+package db
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore contains the LevelDB database that is storing
+// chunk data for all swarm nodes.
+// Closing the GlobalStore with Close method is required to
+// release resources used by the database.
+type GlobalStore struct {
+	db *leveldb.DB
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(path string) (s *GlobalStore, err error) {
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &GlobalStore{
+		db: db,
+	}, nil
+}
+
+// Close releases the resources used by the underlying LevelDB.
+func (s *GlobalStore) Close() error {
+	return s.db.Close()
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) mock.NodeStorer {
+	return &NodeStore{
+		store: s,
+		addr:  addr,
+	}
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		has = false
+	}
+	if !has {
+		return nil, mock.ErrNotFound
+	}
+	data, err = s.db.Get(dataDBKey(key), nil)
+	if err == leveldb.ErrNotFound {
+		err = mock.ErrNotFound
+	}
+	return
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	batch := new(leveldb.Batch)
+	batch.Put(nodeDBKey(addr, key), nil)
+	batch.Put(dataDBKey(key), data)
+	return s.db.Write(batch, nil)
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	has, err := s.db.Has(nodeDBKey(addr, key), nil)
+	if err != nil {
+		has = false
+	}
+	return has
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		batch := new(leveldb.Batch)
+		for _, addr := range c.Addrs {
+			batch.Put(nodeDBKeyHex(addr, hdr.Name), nil)
+		}
+
+		batch.Put(dataDBKey(common.Hex2Bytes(hdr.Name)), c.Data)
+		if err = s.db.Write(batch, nil); err != nil {
+			return n, err
+		}
+
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number fo chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+
+	iter := s.db.NewIterator(util.BytesPrefix(nodeKeyPrefix), nil)
+	defer iter.Release()
+
+	var currentKey string
+	var addrs []common.Address
+
+	saveChunk := func(hexKey string) error {
+		key := common.Hex2Bytes(hexKey)
+
+		data, err := s.db.Get(dataDBKey(key), nil)
+		if err != nil {
+			return err
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: addrs,
+			Data:  data,
+		}); err != nil {
+			return err
+		}
+
+		d := buf.Bytes()
+		hdr := &tar.Header{
+			Name: hexKey,
+			Mode: 0644,
+			Size: int64(len(d)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := tw.Write(d); err != nil {
+			return err
+		}
+		n++
+		return nil
+	}
+
+	for iter.Next() {
+		k := bytes.TrimPrefix(iter.Key(), nodeKeyPrefix)
+		i := bytes.Index(k, []byte("-"))
+		if i < 0 {
+			continue
+		}
+		hexKey := string(k[:i])
+
+		if currentKey == "" {
+			currentKey = hexKey
+		}
+
+		if hexKey != currentKey {
+			if err = saveChunk(currentKey); err != nil {
+				return n, err
+			}
+
+			addrs = addrs[:0]
+		}
+
+		currentKey = hexKey
+		addrs = append(addrs, common.BytesToAddress(k[i:]))
+	}
+
+	if len(addrs) > 0 {
+		if err = saveChunk(currentKey); err != nil {
+			return n, err
+		}
+	}
+
+	return n, err
+}
+
+// NodeStore holds the node address and a reference to the GlobalStore
+// in order to access and store chunk data only for one node.
+type NodeStore struct {
+	store *GlobalStore
+	addr  common.Address
+}
+
+// Get returns chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Get(key []byte) (data []byte, err error) {
+	return n.store.Get(n.addr, key)
+}
+
+// Put saves chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Put(key []byte, data []byte) error {
+	return n.store.Put(n.addr, key, data)
+}
+
+var (
+	nodeKeyPrefix = []byte("node-")
+	dataKeyPrefix = []byte("data-")
+)
+
+// nodeDBKey constructs a database key for key/node mappings.
+func nodeDBKey(addr common.Address, key []byte) []byte {
+	return nodeDBKeyHex(addr, common.Bytes2Hex(key))
+}
+
+// nodeDBKeyHex constructs a database key for key/node mappings
+// using the hexadecimal string representation of the key.
+func nodeDBKeyHex(addr common.Address, hexKey string) []byte {
+	return append(append(nodeKeyPrefix, []byte(hexKey+"-")...), addr[:]...)
+}
+
+// dataDBkey constructs a database key for key/data storage.
+func dataDBKey(key []byte) []byte {
+	return append(dataKeyPrefix, key...)
+}

swarm/storage/mock/db/db_test.go

@@ -0,0 +1,69 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package db
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+func TestDBStore(t *testing.T) {
+	dir, err := ioutil.TempDir("", "mock_"+t.Name())
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir)
+
+	store, err := NewGlobalStore(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store.Close()
+
+	test.MockStore(t, store, 100)
+}
+
+func TestImportExport(t *testing.T) {
+	dir1, err := ioutil.TempDir("", "mock_"+t.Name()+"_exporter")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir1)
+
+	store1, err := NewGlobalStore(dir1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store1.Close()
+
+	dir2, err := ioutil.TempDir("", "mock_"+t.Name()+"_importer")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(dir2)
+
+	store2, err := NewGlobalStore(dir2)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer store2.Close()
+
+	test.ImportExport(t, store1, store2, 100)
+}

swarm/storage/mock/mem/mem.go

@@ -0,0 +1,197 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mem implements a mock store that keeps all chunk data in memory.
+// While it can be used for testing on smaller scales, the main purpose of this
+// package is to provide the simplest reference implementation of a mock store.
+package mem
+
+import (
+	"archive/tar"
+	"bytes"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore stores all chunk data and also keys and node addresses relations.
+// It implements mock.GlobalStore interface.
+type GlobalStore struct {
+	nodes map[string]map[common.Address]struct{}
+	data  map[string][]byte
+	mu    sync.Mutex
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore() *GlobalStore {
+	return &GlobalStore{
+		nodes: make(map[string]map[common.Address]struct{}),
+		data:  make(map[string][]byte),
+	}
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) mock.NodeStorer {
+	return &NodeStore{
+		store: s,
+		addr:  addr,
+	}
+}
+
+// Get returns chunk data if the chunk with key exists for node
+// on address addr.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)][addr]; !ok {
+		return nil, mock.ErrNotFound
+	}
+
+	data, ok := s.data[string(key)]
+	if !ok {
+		return nil, mock.ErrNotFound
+	}
+	return data, nil
+}
+
+// Put saves the chunk data for node with address addr.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if _, ok := s.nodes[string(key)]; !ok {
+		s.nodes[string(key)] = make(map[common.Address]struct{})
+	}
+	s.nodes[string(key)][addr] = struct{}{}
+	s.data[string(key)] = data
+	return nil
+}
+
+// HasKey returns whether a node with addr contains the key.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	_, ok := s.nodes[string(key)][addr]
+	return ok
+}
+
+// Import reads tar archive from a reader that contains exported chunk data.
+// It returns the number of chunks imported and an error.
+func (s *GlobalStore) Import(r io.Reader) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return n, err
+		}
+
+		data, err := ioutil.ReadAll(tr)
+		if err != nil {
+			return n, err
+		}
+
+		var c mock.ExportedChunk
+		if err = json.Unmarshal(data, &c); err != nil {
+			return n, err
+		}
+
+		addrs := make(map[common.Address]struct{})
+		for _, a := range c.Addrs {
+			addrs[a] = struct{}{}
+		}
+
+		key := string(common.Hex2Bytes(hdr.Name))
+		s.nodes[key] = addrs
+		s.data[key] = c.Data
+		n++
+	}
+	return n, err
+}
+
+// Export writes to a writer a tar archive with all chunk data from
+// the store. It returns the number fo chunks exported and an error.
+func (s *GlobalStore) Export(w io.Writer) (n int, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	tw := tar.NewWriter(w)
+	defer tw.Close()
+
+	buf := bytes.NewBuffer(make([]byte, 0, 1024))
+	encoder := json.NewEncoder(buf)
+	for key, addrs := range s.nodes {
+		al := make([]common.Address, 0, len(addrs))
+		for a := range addrs {
+			al = append(al, a)
+		}
+
+		buf.Reset()
+		if err = encoder.Encode(mock.ExportedChunk{
+			Addrs: al,
+			Data:  s.data[key],
+		}); err != nil {
+			return n, err
+		}
+
+		data := buf.Bytes()
+		hdr := &tar.Header{
+			Name: common.Bytes2Hex([]byte(key)),
+			Mode: 0644,
+			Size: int64(len(data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return n, err
+		}
+		if _, err := tw.Write(data); err != nil {
+			return n, err
+		}
+		n++
+	}
+	return n, err
+}
+
+// NodeStore holds the node address and a reference to the GlobalStore
+// in order to access and store chunk data only for one node.
+type NodeStore struct {
+	store *GlobalStore
+	addr  common.Address
+}
+
+// Get returns chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Get(key []byte) (data []byte, err error) {
+	return n.store.Get(n.addr, key)
+}
+
+// Put saves chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Put(key []byte, data []byte) error {
+	return n.store.Put(n.addr, key, data)
+}

swarm/storage/mock/mem/mem_test.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package mem
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+func TestMemStore(t *testing.T) {
+	test.MockStore(t, NewGlobalStore(), 100)
+}
+
+func TestImportExport(t *testing.T) {
+	test.ImportExport(t, NewGlobalStore(), NewGlobalStore(), 100)
+}

swarm/storage/mock/mock.go

@@ -0,0 +1,93 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package mock defines types that are used by different implementations
+// of mock storages.
+//
+// Implementations of mock storages are located in directories
+// under this package:
+//
+//  - db - LevelDB backend
+//  - mem - in memory map backend
+//  - rpc - RPC client that can connect to other backends
+//
+// Mock storages can implement Importer and Exporter interfaces
+// for importing and exporting all chunk data that they contain.
+// The exported file is a tar archive with all files named by
+// hexadecimal representations of chunk keys and with content
+// with JSON-encoded ExportedChunk structure. Exported format
+// should be preserved across all mock store implementations.
+package mock
+
+import (
+	"errors"
+	"io"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// ErrNotFound is in all NodeStorer implementations
+// to indicate that the chunk is not found.
+var ErrNotFound = errors.New("not found")
+
+// GlobalStorer defines methods for mock db store
+// that stores chunk data for all swarm nodes.
+// It is used in tests to construct mock NodeStores
+// for swarm nodes and to track and validate chunks.
+type GlobalStorer interface {
+	Get(addr common.Address, key []byte) (data []byte, err error)
+	Put(addr common.Address, key []byte, data []byte) error
+	HasKey(addr common.Address, key []byte) bool
+	// NewNodeStore creates an instance of NodeStorer
+	// to be used by a single swarm node with
+	// address addr.
+	NewNodeStore(addr common.Address) NodeStorer
+}
+
+// NodeStorer defines methods that are required
+// for accessing and storing chunk data.
+// It is used for baypassing chunk data storing in
+// storage.DbStore.
+type NodeStorer interface {
+	Get(key []byte) (data []byte, err error)
+	Put(key []byte, data []byte) error
+}
+
+// Importer defines method for importing mock store data
+// from an exported tar archive.
+type Importer interface {
+	Import(r io.Reader) (n int, err error)
+}
+
+// Exporter defines method for exporting mock store data
+// to a tar archive.
+type Exporter interface {
+	Export(w io.Writer) (n int, err error)
+}
+
+// ImportExporter is an interface for importing and exporting
+// mock store data to and from a tar archive.
+type ImportExporter interface {
+	Importer
+	Exporter
+}
+
+// ExportedChunk is the structure that is saved in tar archive for
+// each chunk as JSON-encoded bytes.
+type ExportedChunk struct {
+	Data  []byte           `json:"d"`
+	Addrs []common.Address `json:"a"`
+}

swarm/storage/mock/rpc/rpc.go

@@ -0,0 +1,106 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package rpc implements an RPC client that connect to a centralized mock store.
+// Centralazied mock store can be any other mock store implementation that is
+// registered to Ethereum RPC server under mockStore name. Methods that defines
+// mock.GlobalStore are the same that are used by RPC. Example:
+//
+//   server := rpc.NewServer()
+//   server.RegisterName("mockStore", mem.NewGlobalStore())
+package rpc
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// GlobalStore is rpc.Client that connects to a centralized mock store.
+// Closing GlobalStore instance is required to release RPC client resources.
+type GlobalStore struct {
+	client *rpc.Client
+}
+
+// NewGlobalStore creates a new instance of GlobalStore.
+func NewGlobalStore(client *rpc.Client) *GlobalStore {
+	return &GlobalStore{
+		client: client,
+	}
+}
+
+// Close closes RPC client.
+func (s *GlobalStore) Close() error {
+	s.client.Close()
+	return nil
+}
+
+// NewNodeStore returns a new instance of NodeStore that retrieves and stores
+// chunk data only for a node with address addr.
+func (s *GlobalStore) NewNodeStore(addr common.Address) mock.NodeStorer {
+	return &NodeStore{
+		store: s,
+		addr:  addr,
+	}
+}
+
+// Get calls a Get method to RPC server.
+func (s *GlobalStore) Get(addr common.Address, key []byte) (data []byte, err error) {
+	err = s.client.Call(&data, "mockStore_get", addr, key)
+	if err != nil && err.Error() == "not found" {
+		// pass the mock package value of error instead an rpc error
+		return data, mock.ErrNotFound
+	}
+	return data, err
+}
+
+// Put calls a Put method to RPC server.
+func (s *GlobalStore) Put(addr common.Address, key []byte, data []byte) error {
+	err := s.client.Call(nil, "mockStore_put", addr, key, data)
+	return err
+}
+
+// HasKey calls a HasKey method to RPC server.
+func (s *GlobalStore) HasKey(addr common.Address, key []byte) bool {
+	var has bool
+	if err := s.client.Call(&has, "mockStore_hasKey", addr, key); err != nil {
+		log.Error(fmt.Sprintf("mock store HasKey: addr %s, key %064x: %v", addr, key, err))
+		return false
+	}
+	return has
+}
+
+// NodeStore holds the node address and a reference to the GlobalStore
+// in order to access and store chunk data only for one node.
+type NodeStore struct {
+	store *GlobalStore
+	addr  common.Address
+}
+
+// Get returns chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Get(key []byte) (data []byte, err error) {
+	return n.store.Get(n.addr, key)
+}
+
+// Put saves chunk data for a key for a node that has the address
+// provided on NodeStore initialization.
+func (n *NodeStore) Put(key []byte, data []byte) error {
+	return n.store.Put(n.addr, key, data)
+}

swarm/storage/mock/rpc/rpc_test.go

@@ -0,0 +1,39 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rpc
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/mem"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock/test"
+)
+
+func TestRPCStore(t *testing.T) {
+	serverStore := mem.NewGlobalStore()
+
+	server := rpc.NewServer()
+	if err := server.RegisterName("mockStore", serverStore); err != nil {
+		t.Fatal(err)
+	}
+
+	store := NewGlobalStore(rpc.DialInProc(server))
+	defer store.Close()
+
+	test.MockStore(t, store, 100)
+}

swarm/storage/mock/test/test.go

@@ -0,0 +1,186 @@
+// Copyright 2018 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package test provides functions that are used for testing
+// GlobalStorer and NodeStorer implementations.
+package test
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"strconv"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
+)
+
+// MockStore creates NodeStorer instances from provided GlobalStorer,
+// each one with a unique address, stores different chunks on them
+// and checks if they are retrievable or not on all nodes.
+// Attribute n defines the number of NodeStorers that will be created.
+func MockStore(t *testing.T, globalStore mock.GlobalStorer, n int) {
+	t.Run("GlobalStore", func(t *testing.T) {
+		addrs := make([]common.Address, n)
+		for i := 0; i < n; i++ {
+			addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+		}
+
+		for i, addr := range addrs {
+			key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			globalStore.Put(addr, key, data)
+
+			for _, cAddr := range addrs {
+				cData, err := globalStore.Get(cAddr, key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+
+	t.Run("NodeStore", func(t *testing.T) {
+		nodes := make(map[common.Address]mock.NodeStorer)
+		for i := 0; i < n; i++ {
+			addr := common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+			nodes[addr] = globalStore.NewNodeStore(addr)
+		}
+
+		i := 0
+		for addr, store := range nodes {
+			i++
+			key := storage.Key(append(addr[:], []byte(fmt.Sprintf("%x", i))...))
+			data := []byte(strconv.FormatInt(int64(i)+1, 16))
+			data = append(data, make([]byte, 4096-len(data))...)
+			store.Put(key, data)
+
+			for cAddr, cStore := range nodes {
+				cData, err := cStore.Get(key)
+				if cAddr == addr {
+					if err != nil {
+						t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+					}
+					if !bytes.Equal(data, cData) {
+						t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+					}
+					if !globalStore.HasKey(cAddr, key) {
+						t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+					}
+				} else {
+					if err != mock.ErrNotFound {
+						t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+					}
+					if len(cData) > 0 {
+						t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+					}
+					if globalStore.HasKey(cAddr, key) {
+						t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+					}
+				}
+			}
+		}
+	})
+}
+
+// ImportExport saves chunks to the outStore, exports them to the tar archive,
+// imports tar archive to the inStore and checks if all chunks are imported correctly.
+func ImportExport(t *testing.T, outStore, inStore mock.GlobalStorer, n int) {
+	exporter, ok := outStore.(mock.Exporter)
+	if !ok {
+		t.Fatal("outStore does not implement mock.Exporter")
+	}
+	importer, ok := inStore.(mock.Importer)
+	if !ok {
+		t.Fatal("inStore does not implement mock.Importer")
+	}
+	addrs := make([]common.Address, n)
+	for i := 0; i < n; i++ {
+		addrs[i] = common.HexToAddress(strconv.FormatInt(int64(i)+1, 16))
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		outStore.Put(addr, key, data)
+	}
+
+	r, w := io.Pipe()
+	defer r.Close()
+
+	go func() {
+		defer w.Close()
+		if _, err := exporter.Export(w); err != nil {
+			t.Fatalf("export: %v", err)
+		}
+	}()
+
+	if _, err := importer.Import(r); err != nil {
+		t.Fatalf("import: %v", err)
+	}
+
+	for i, addr := range addrs {
+		key := storage.Key(append(addr[:], []byte(strconv.FormatInt(int64(i)+1, 16))...))
+		data := []byte(strconv.FormatInt(int64(i)+1, 16))
+		data = append(data, make([]byte, 4096-len(data))...)
+		for _, cAddr := range addrs {
+			cData, err := inStore.Get(cAddr, key)
+			if cAddr == addr {
+				if err != nil {
+					t.Fatalf("get data from store %s key %s: %v", cAddr.Hex(), key.Hex(), err)
+				}
+				if !bytes.Equal(data, cData) {
+					t.Fatalf("data on store %s: expected %x, got %x", cAddr.Hex(), data, cData)
+				}
+				if !inStore.HasKey(cAddr, key) {
+					t.Fatalf("expected key %s on global store for node %s, but it was not found", key.Hex(), cAddr.Hex())
+				}
+			} else {
+				if err != mock.ErrNotFound {
+					t.Fatalf("expected error from store %s: %v, got %v", cAddr.Hex(), mock.ErrNotFound, err)
+				}
+				if len(cData) > 0 {
+					t.Fatalf("data on store %s: expected nil, got %x", cAddr.Hex(), cData)
+				}
+				if inStore.HasKey(cAddr, key) {
+					t.Fatalf("not expected key %s on global store for node %s, but it was found", key.Hex(), cAddr.Hex())
+				}
+			}
+		}
+	}
+}

swarm/swarm.go

@@ -41,6 +41,7 @@ import (
 	"github.com/ethereum/go-ethereum/swarm/network"
 	"github.com/ethereum/go-ethereum/swarm/pss"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+	"github.com/ethereum/go-ethereum/swarm/storage/mock"
 )
 
 // the swarm stack
@@ -79,7 +80,9 @@ func (self *Swarm) API() *SwarmAPI {
 
 // creates a new swarm service instance
 // implements node.Service
-func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool) (self *Swarm, err error) {
+// If mockStore is not nil, it will be used as the storage for chunk data.
+// MockStore should be used only for testing.
+func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *ethclient.Client, config *api.Config, swapEnabled, syncEnabled bool, cors string, pssEnabled bool, mockStore mock.NodeStorer) (self *Swarm, err error) {
 	if bytes.Equal(common.FromHex(config.PublicKey), storage.ZeroKey) {
 		return nil, fmt.Errorf("empty public key")
 	}
@@ -97,7 +100,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, ensClient *e
 	log.Debug(fmt.Sprintf("Setting up Swarm service components"))
 
 	hash := storage.MakeHashFunc(config.ChunkerParams.Hash)
-	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams)
+	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, mockStore)
 	if err != nil {
 		return
 	}