Abstraction of chunk data validation (#337)

* swarm/storage: Replace "trusted" settings with validator

* swarm/storage: Remove commented code

* swarm: Update storage calls revised for abstract validator

* swarm/storage: Connect resource validator to chunk validation

* swarm/storage: WIP storeparams generalization + validator abstract

* swarm/storage: Remove modularity in resource update validation

* swarm/storage: Move validation chunkstore iface to ldbstore

* swarm: Add sequential chunk validator

First checks content addressing, if it fails then
checks resource validation.

* swarm: Rebase on swarm-network-rewrite

* swarm/storage: Update toml package + omit func types in toml marshal

* swarm/storage: Rebase after encryption merge

* swarm/storage, swarm/network: Move validator to localstore

* cmd/swarm: Remove radius flag

* swarm: Arrange instantiations in correct order in swarm/swarm.go

* swarm/storage: Fix validator concurrency

* cmd/swarm: Make ExportImport test pass.

(BaseKey param for localstore set erroneously)

* swarm/storage, cmd/swarm: Remove redundant logs

* swarm/storage: Make memstore disabled on capacity 0

* swarm/storage: Simplify Chunk Validator objects/interface

* swarm/storage: Add validator tests

* swarm/storage: Refactor validators to array

* swarm/storage: Revert hasher instantation + validator logic loop

* swarm/storage: Fix validator iteration logic

Add error type for chunks

* swarm/storage: remove commented code; improve comments and tests with errors

cmd/swarm/config.go

@@ -76,7 +76,6 @@ const (
 	SWARM_ENV_STORE_PATH           = "SWARM_STORE_PATH"
 	SWARM_ENV_STORE_CAPACITY       = "SWARM_STORE_CAPACITY"
 	SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
-	SWARM_ENV_STORE_RADIUS         = "SWARM_STORE_RADIUS"
 	GETH_ENV_DATADIR               = "GETH_DATADIR"
 )
 
@@ -237,19 +236,15 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 	}
 
 	if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
-		currentConfig.StoreParams.ChunkDbPath = storePath
+		currentConfig.LocalStoreParams.ChunkDbPath = storePath
 	}
 
 	if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
-		currentConfig.StoreParams.DbCapacity = storeCapacity
+		currentConfig.LocalStoreParams.DbCapacity = storeCapacity
 	}
 
 	if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
-		currentConfig.StoreParams.CacheCapacity = storeCacheCapacity
+		currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity
-	}
-
-	if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 {
-		currentConfig.StoreParams.Radius = storeRadius
 	}
 
 	return currentConfig

cmd/swarm/config_test.go

@@ -156,7 +156,7 @@ func TestConfigFileOverrides(t *testing.T) {
 	defaultConf.PssEnabled = true
 	defaultConf.NetworkId = 54
 	defaultConf.Port = httpPort
-	defaultConf.StoreParams.DbCapacity = 9000000
+	defaultConf.DbCapacity = 9000000
 	defaultConf.HiveParams.KeepAliveInterval = 6000000000
 	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
 	//defaultConf.SyncParams.KeyBufferSize = 512
@@ -232,7 +232,7 @@ func TestConfigFileOverrides(t *testing.T) {
 		t.Fatal("Expected Pss to be enabled, but is false")
 	}
 
-	if info.StoreParams.DbCapacity != 9000000 {
+	if info.DbCapacity != 9000000 {
 		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
 	}
 
@@ -368,7 +368,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 	defaultConf.PssEnabled = false
 	defaultConf.NetworkId = 54
 	defaultConf.Port = "8588"
-	defaultConf.StoreParams.DbCapacity = 9000000
+	defaultConf.DbCapacity = 9000000
 	defaultConf.HiveParams.KeepAliveInterval = 6000000000
 	defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
 	//defaultConf.SyncParams.KeyBufferSize = 512
@@ -378,10 +378,12 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 		t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
 	}
 	//write file
-	f, err := ioutil.TempFile("", "testconfig.toml")
+	fname := "testconfig.toml"
+	f, err := ioutil.TempFile("", fname)
 	if err != nil {
 		t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
 	}
+	defer os.Remove(fname)
 	//write file
 	_, err = f.WriteString(string(out))
 	if err != nil {
@@ -446,8 +448,8 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
 		t.Fatal("Expected Sync to be enabled, but is false")
 	}
 
-	if info.StoreParams.DbCapacity != 9000000 {
+	if info.LocalStoreParams.DbCapacity != 9000000 {
-		t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
+		t.Fatalf("Expected Capacity to be %d, got %d", 9000000, info.LocalStoreParams.DbCapacity)
 	}
 
 	if info.HiveParams.KeepAliveInterval != 6000000000 {

cmd/swarm/db.go

@@ -112,6 +112,6 @@ func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return nil, fmt.Errorf("invalid chunkdb path: %s", err)
 	}
-	hash := storage.MakeHashFunc("SHA3")
+	storeParams := storage.NewLDBStoreParams(path, 10000000, nil, nil)
-	return storage.NewLDBStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) })
+	return storage.NewLDBStore(storeParams)
 }

cmd/swarm/main.go

@@ -169,11 +169,6 @@ var (
 		Usage:  "Number of recent chunks cached in memory (default 5000)",
 		EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
 	}
-	SwarmStoreRadius = cli.IntFlag{
-		Name:   "store.radius",
-		Usage:  "Minimum proximity order (number of identical prefix bits of address key) for chunks to warrant storage (default 0)",
-		EnvVar: SWARM_ENV_STORE_RADIUS,
-	}
 
 	// the following flags are deprecated and should be removed in the future
 	DeprecatedEthAPIFlag = cli.StringFlag{
@@ -385,7 +380,6 @@ Remove corrupt entries from a local chunk database.
 		SwarmStorePath,
 		SwarmStoreCapacity,
 		SwarmStoreCacheCapacity,
-		SwarmStoreRadius,
 		//deprecated flags
 		DeprecatedEthAPIFlag,
 		DeprecatedEnsAddrFlag,

swarm/api/api.go

@@ -617,7 +617,7 @@ func (self *Api) ResourceUpdate(ctx context.Context, name string, data []byte) (
 }
 
 func (self *Api) ResourceHashSize() int {
-	return self.resource.HashSize()
+	return self.resource.HashSize
 }
 
 func (self *Api) ResourceIsValidated() bool {

swarm/api/config.go

@@ -42,8 +42,8 @@ const (
 // allow several bzz nodes running in parallel
 type Config struct {
 	// serialised/persisted fields
-	*storage.StoreParams
 	*storage.DPAParams
+	*storage.LocalStoreParams
 	*network.HiveParams
 	Swap *swap.SwapParams
 	//*network.SyncParams
@@ -72,9 +72,9 @@ type Config struct {
 func NewConfig() (self *Config) {
 
 	self = &Config{
-		StoreParams: storage.NewDefaultStoreParams(),
+		LocalStoreParams: storage.NewDefaultLocalStoreParams(),
-		DPAParams:   storage.NewDPAParams(),
+		DPAParams:        storage.NewDPAParams(),
-		HiveParams:  network.NewHiveParams(),
+		HiveParams:       network.NewHiveParams(),
 		//SyncParams:    network.NewDefaultSyncParams(),
 		Swap:            swap.NewDefaultSwapParams(),
 		ListenAddr:      DefaultHTTPListenAddr,
@@ -117,8 +117,9 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
 	if self.SwapEnabled {
 		self.Swap.Init(self.Contract, prvKey)
 	}
+
 	self.privateKey = prvKey
-	self.StoreParams.Init(self.Path)
+	self.LocalStoreParams.Init(self.Path)
 }
 
 func (self *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) {

swarm/api/config_test.go

@@ -36,6 +36,7 @@ func TestConfig(t *testing.T) {
 	one := NewConfig()
 	two := NewConfig()
 
+	one.LocalStoreParams = two.LocalStoreParams
 	if equal := reflect.DeepEqual(one, two); !equal {
 		t.Fatal("Two default configs are not equal")
 	}
@@ -52,7 +53,7 @@ func TestConfig(t *testing.T) {
 	if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled {
 		t.Fatal("Failed to correctly initialize SwapParams")
 	}
-	if one.StoreParams.ChunkDbPath == one.Path {
+	if one.ChunkDbPath == one.Path {
 		t.Fatal("Failed to correctly initialize StoreParams")
 	}
 }

swarm/api/http/server.go

@@ -481,13 +481,13 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
 	code := 0
 	defaultErr := fmt.Errorf("%s: %v", supErr, err)
 	rsrcErr, ok := err.(*storage.ResourceError)
-	if !ok {
+	if !ok && rsrcErr != nil {
 		code = rsrcErr.Code()
 	}
 	switch code {
 	case storage.ErrInvalidValue:
 		return http.StatusBadRequest, defaultErr
-	case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn:
+	case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit:
 		return http.StatusNotFound, defaultErr
 	case storage.ErrUnauthorized, storage.ErrInvalidSignature:
 		return http.StatusUnauthorized, defaultErr

swarm/api/http/server_test.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	swarm "github.com/ethereum/go-ethereum/swarm/api/client"
@@ -37,11 +38,9 @@ import (
 )
 
 func init() {
-	verbose := flag.Bool("v", false, "verbose")
+	loglevel := flag.Int("loglevel", 2, "loglevel")
 	flag.Parse()
-	if *verbose {
+	log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
-		log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
-	}
 }
 
 type resourceResponse struct {
@@ -55,10 +54,8 @@ func TestBzzResource(t *testing.T) {
 	defer srv.Close()
 
 	// our mutable resource "name"
-	keybytes := []byte("foo")
+	keybytes := "foo"
-	srv.Hasher.Reset()
+	keybyteshash := ens.EnsNode(keybytes)
-	srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes)))
-	keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil))
 
 	// data of update 1
 	databytes := make([]byte, 666)
@@ -68,7 +65,7 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// creates resource and sets update 1
-	url := fmt.Sprintf("%s/bzz-resource:/%x/13", srv.URL, keybytes)
+	url := fmt.Sprintf("%s/bzz-resource:/%s/13", srv.URL, []byte(keybytes))
 	resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes))
 	if err != nil {
 		t.Fatal(err)
@@ -86,8 +83,8 @@ func TestBzzResource(t *testing.T) {
 	if err != nil {
 		t.Fatalf("data %s could not be unmarshaled: %v", b, err)
 	}
-	if rsrcResp.Update.Hex() != keybyteshash {
+	if !bytes.Equal(rsrcResp.Update, keybyteshash.Bytes()) {
-		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash, rsrcResp.Resource)
+		t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash.Hex(), rsrcResp.Update.Hex())
 	}
 
 	// get manifest
@@ -131,8 +128,16 @@ func TestBzzResource(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// get non-existent name
+	url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
+	resp, err = http.Get(url)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resp.Body.Close()
+
 	// get latest update (1.1) through resource directly
-	url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
+	url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
 	resp, err = http.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -150,7 +155,7 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// update 2
-	url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
+	url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
 	data := []byte("foo")
 	resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data))
 	if err != nil {
@@ -162,7 +167,7 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// get latest update (1.2) through resource directly
-	url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
+	url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
 	resp, err = http.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -180,7 +185,7 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// get latest update (1.2) with specified period
-	url = fmt.Sprintf("%s/bzz-resource:/%x/1", srv.URL, keybytes)
+	url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, keybytes)
 	resp, err = http.Get(url)
 	if err != nil {
 		t.Fatal(err)
@@ -198,7 +203,7 @@ func TestBzzResource(t *testing.T) {
 	}
 
 	// get first update (1.1) with specified period and version
-	url = fmt.Sprintf("%s/bzz-resource:/%x/1/1", srv.URL, keybytes)
+	url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, keybytes)
 	resp, err = http.Get(url)
 	if err != nil {
 		t.Fatal(err)

swarm/fuse/swarmfs_test.go

@@ -21,6 +21,7 @@ package fuse
 import (
 	"bytes"
 	"crypto/rand"
+	"flag"
 	"io"
 	"io/ioutil"
 	"os"
@@ -29,8 +30,23 @@ import (
 
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/storage"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	colorable "github.com/mattn/go-colorable"
 )
 
+var (
+	loglevel = flag.Int("loglevel", 4, "verbosity of logs")
+	rawlog   = flag.Bool("rawlog", false, "turn off terminal formatting in logs")
+)
+
+func init() {
+	flag.Parse()
+	log.PrintOrigins(true)
+	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog))))
+}
+
 type fileInfo struct {
 	perm     uint64
 	uid      int

swarm/network/stream/common_test.go

@@ -133,7 +133,11 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora
 		os.RemoveAll(datadir)
 	}
 
-	localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+	params := storage.NewDefaultLocalStoreParams()
+	params.Init(datadir)
+	params.BaseKey = addr.Over()
+
+	localStore, err := storage.NewTestLocalStoreForAddr(params)
 	if err != nil {
 		return nil, nil, nil, removeDataDir, err
 	}

swarm/network/stream/delivery.go

@@ -138,7 +138,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e
 		if created {
 			if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil {
 				log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err)
-				chunk.SetErrored(true)
+				chunk.SetErrored(storage.ErrChunkForward)
 				return nil
 			}
 		}
@@ -149,10 +149,10 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e
 			select {
 			case <-chunk.ReqC:
 			case <-t.C:
-				chunk.SetErrored(true)
+				chunk.SetErrored(storage.ErrChunkTimeout)
 				return
 			}
-			chunk.SetErrored(false)
+			chunk.SetErrored(nil)
 
 			if req.SkipCheck {
 				err := sp.Deliver(chunk, s.priority)
@@ -209,6 +209,12 @@ R:
 		chunk.SData = req.SData
 		d.db.Put(chunk)
 		chunk.WaitToStore()
+		err = chunk.GetErrored()
+		if err != nil {
+			if err == storage.ErrChunkInvalid {
+				req.peer.Drop(err)
+			}
+		}
 		close(chunk.ReqC)
 	}
 }

swarm/network/stream/snapshot_sync_test.go

@@ -662,7 +662,10 @@ func createTestLocalStorageForId(id discover.NodeID, addr *network.BzzAddr) (sto
 	}
 	datadirs[id] = datadir
 	var store storage.ChunkStore
-	store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+	params := storage.NewDefaultLocalStoreParams()
+	params.ChunkDbPath = datadir
+	params.BaseKey = addr.Over()
+	store, err = storage.NewTestLocalStoreForAddr(params)
 	if err != nil {
 		return nil, err
 	}

swarm/network/stream/testing/testing.go

@@ -55,7 +55,10 @@ func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) {
 			break
 		}
 		var store storage.ChunkStore
-		store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
+		params := storage.NewDefaultLocalStoreParams()
+		params.Init(datadir)
+		params.BaseKey = addr.Over()
+		store, err = storage.NewTestLocalStoreForAddr(params)
 		if err != nil {
 			break
 		}

swarm/storage/dpa.go

@@ -17,7 +17,6 @@
 package storage
 
 import (
-	"errors"
 	"io"
 	"time"
 )
@@ -40,8 +39,6 @@ const (
 )
 
 var (
-	ErrChunkNotFound = errors.New("chunk not found")
-	ErrFetching      = errors.New("chunk still fetching")
 	// timeout interval before retrieval is timed out
 	searchTimeout = 3 * time.Second
 )
@@ -63,18 +60,14 @@ func NewDPAParams() *DPAParams {
 
 // for testing locally
 func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) {
-
+	params := NewDefaultLocalStoreParams()
-	hash := MakeHashFunc("SHA3")
+	params.Init(datadir)
-
+	localStore, err := NewLocalStore(params, nil)
-	dbStore, err := NewLDBStore(datadir, hash, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
 	if err != nil {
 		return nil, err
 	}
-
+	localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(SHA3Hash)))
-	return NewDPA(&LocalStore{
+	return NewDPA(localStore, NewDPAParams()), nil
-		memStore: NewMemStore(dbStore, singletonSwarmCacheCapacity),
-		DbStore:  dbStore,
-	}, NewDPAParams()), nil
 }
 
 func NewDPA(store ChunkStore, params *DPAParams) *DPA {

swarm/storage/dpa_test.go

@@ -32,14 +32,15 @@ func TestDPArandom(t *testing.T) {
 }
 
 func testDpaRandom(toEncrypt bool, t *testing.T) {
-	tdb, err := newTestDbStore(false)
+	tdb, err := newTestDbStore(false, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
 	defer tdb.close()
 	db := tdb.LDBStore
 	db.setCapacity(50000)
-	memStore := NewMemStore(db, defaultCacheCapacity)
+	storeParams := NewStoreParams(defaultCacheCapacity, nil, nil)
+	memStore := NewMemStore(storeParams, db)
 	localStore := &LocalStore{
 		memStore: memStore,
 		DbStore:  db,
@@ -71,7 +72,7 @@ func testDpaRandom(toEncrypt bool, t *testing.T) {
 	}
 	ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
 	ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
-	localStore.memStore = NewMemStore(db, defaultCacheCapacity)
+	localStore.memStore = NewMemStore(storeParams, db)
 	resultReader, isEncrypted = dpa.Retrieve(key)
 	if isEncrypted != toEncrypt {
 		t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
@@ -97,13 +98,15 @@ func TestDPA_capacity(t *testing.T) {
 }
 
 func testDPA_capacity(toEncrypt bool, t *testing.T) {
-	tdb, err := newTestDbStore(false)
+	tdb, err := newTestDbStore(false, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
 	defer tdb.close()
 	db := tdb.LDBStore
-	memStore := NewMemStore(db, 0)
+	storeParams := NewStoreParams(0, nil, nil)
+	storeParams.CacheCapacity = 10000000
+	memStore := NewMemStore(storeParams, db)
 	localStore := &LocalStore{
 		memStore: memStore,
 		DbStore:  db,

swarm/storage/error.go

@@ -1,7 +1,12 @@
 package storage
 
+import (
+	"errors"
+)
+
 const (
-	ErrNotFound = iota
+	ErrInit = iota
+	ErrNotFound
 	ErrIO
 	ErrUnauthorized
 	ErrInvalidValue
@@ -12,3 +17,12 @@ const (
 	ErrPeriodDepth
 	ErrCnt
 )
+
+var (
+	ErrChunkNotFound    = errors.New("chunk not found")
+	ErrFetching         = errors.New("chunk still fetching")
+	ErrChunkInvalid     = errors.New("invalid chunk")
+	ErrChunkForward     = errors.New("cannot forward")
+	ErrChunkUnavailable = errors.New("chunk unavailable")
+	ErrChunkTimeout     = errors.New("timeout")
+)

swarm/storage/ldbstore.go

@@ -75,6 +75,30 @@ type gcItem struct {
 	idxKey []byte
 }
 
+type LDBStoreParams struct {
+	*StoreParams
+	Path string
+	Po   func(Key) uint8
+}
+
+// NewLDBStoreParams constructs LDBStoreParams with the specified values.
+// if 0 is set for capacity or nil for hash, and basekey is specified, default values for these params will be used
+// path has no default value
+func NewLDBStoreParams(path string, capacity uint64, hash SwarmHasher, basekey []byte) *LDBStoreParams {
+	if hash == nil {
+		hash = MakeHashFunc(SHA3Hash)
+	}
+	if capacity == 0 {
+		capacity = singletonSwarmDbCapacity
+	}
+	storeparams := NewStoreParams(capacity, hash, basekey)
+	return &LDBStoreParams{
+		StoreParams: storeparams,
+		Path:        path,
+		Po:          func(k Key) (ret uint8) { return uint8(Proximity(storeparams.BaseKey[:], k[:])) },
+	}
+}
+
 type LDBStore struct {
 	db *LDBDatabase
 
@@ -92,7 +116,6 @@ type LDBStore struct {
 	batchesC chan struct{}
 	batch    *leveldb.Batch
 	lock     sync.RWMutex
-	trusted  bool // if hash integity check is to be performed (for testing only)
 
 	// Functions encodeDataFunc is used to bypass
 	// the default functionality of DbStore with
@@ -107,9 +130,9 @@ type LDBStore struct {
 // TODO: Instead of passing the distance function, just pass the address from which distances are calculated
 // to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing
 // a function different from the one that is actually used.
-func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *LDBStore, err error) {
+func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) {
 	s = new(LDBStore)
-	s.hashfunc = hash
+	s.hashfunc = params.Hash
 
 	s.batchC = make(chan bool)
 	s.batchesC = make(chan struct{}, 1)
@@ -118,13 +141,13 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui
 	// associate encodeData with default functionality
 	s.encodeDataFunc = encodeData
 
-	s.db, err = NewLDBDatabase(path)
+	s.db, err = NewLDBDatabase(params.Path)
 	if err != nil {
 		return nil, err
 	}
 
-	s.po = po
+	s.po = params.Po
-	s.setCapacity(capacity)
+	s.setCapacity(params.DbCapacity)
 
 	s.gcStartPos = make([]byte, 1)
 	s.gcStartPos[0] = kpIndex
@@ -156,15 +179,11 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui
 	return s, nil
 }
 
-func (self *LDBStore) SetTrusted() {
-	self.trusted = true
-}
-
 // NewMockDbStore creates a new instance of DbStore with
 // mockStore set to a provided value. If mockStore argument is nil,
 // this function behaves exactly as NewDbStore.
-func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *LDBStore, err error) {
+func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) {
-	s, err = NewLDBStore(path, hash, capacity, po)
+	s, err = NewLDBStore(params)
 	if err != nil {
 		return nil, err
 	}
@@ -413,7 +432,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
 			continue
 		}
 
-		key, err := hex.DecodeString(hdr.Name)
+		keybytes, err := hex.DecodeString(hdr.Name)
 		if err != nil {
 			log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err)
 			continue
@@ -423,6 +442,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
 		if err != nil {
 			return count, err
 		}
+		key := Key(keybytes)
 		chunk := NewChunk(key, nil)
 		chunk.SData = data[32:]
 		s.Put(chunk)
@@ -700,19 +720,6 @@ func (s *LDBStore) get(key Key) (chunk *Chunk, err error) {
 			}
 		}
 
-		if !s.trusted {
-			data_mod := data[32:]
-			hasher := s.hashfunc()
-			hasher.Write(data_mod)
-			hash := hasher.Sum(nil)
-
-			if !bytes.Equal(hash, key) {
-				log.Error("apparent key/hash mismatch", "hash", hash, "key", key[:])
-				s.delete(indx.Idx, getIndexKey(key), s.po(key))
-				log.Error("invalid chunk in database.")
-			}
-		}
-
 		chunk = NewChunk(key, nil)
 		chunk.markAsStored()
 		decodeData(data, chunk)

swarm/storage/ldbstore_test.go

@@ -34,21 +34,24 @@ type testDbStore struct {
 	dir string
 }
 
-func newTestDbStore(mock bool) (*testDbStore, error) {
+func newTestDbStore(mock bool, trusted bool) (*testDbStore, error) {
 	dir, err := ioutil.TempDir("", "bzz-storage-test")
 	if err != nil {
 		return nil, err
 	}
 
 	var db *LDBStore
+	params := NewLDBStoreParams(dir, defaultDbCapacity, nil, nil)
+	params.Po = testPoFunc
+
 	if mock {
 		globalStore := mem.NewGlobalStore()
 		addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
 		mockStore := globalStore.NewNodeStore(addr)
 
-		db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc, mockStore)
+		db, err = NewMockDbStore(params, mockStore)
 	} else {
-		db, err = NewLDBStore(dir, MakeHashFunc(SHA3Hash), defaultDbCapacity, testPoFunc)
+		db, err = NewLDBStore(params)
 	}
 
 	return &testDbStore{db, dir}, err
@@ -68,17 +71,16 @@ func (db *testDbStore) close() {
 }
 
 func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) {
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
 	defer db.close()
-	db.trusted = true
 	testStoreRandom(db, processors, n, chunksize, t)
 }
 
 func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) {
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
@@ -135,7 +137,7 @@ func TestMockDbStoreCorrect_8_5k(t *testing.T) {
 }
 
 func testDbStoreNotFound(t *testing.T, mock bool) {
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
@@ -166,7 +168,7 @@ func testIterator(t *testing.T, mock bool) {
 		chunks = append(chunks, NewChunk(nil, nil))
 	}
 
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, false)
 	if err != nil {
 		t.Fatalf("init dbStore failed: %v", err)
 	}
@@ -221,22 +223,20 @@ func TestMockIterator(t *testing.T) {
 }
 
 func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) {
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)
 	}
 	defer db.close()
-	db.trusted = true
 	benchmarkStorePut(db, processors, n, chunksize, b)
 }
 
 func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) {
-	db, err := newTestDbStore(mock)
+	db, err := newTestDbStore(mock, true)
 	if err != nil {
 		b.Fatalf("init dbStore failed: %v", err)
 	}
 	defer db.close()
-	db.trusted = true
 	benchmarkStoreGet(db, processors, n, chunksize, b)
 }
 

swarm/storage/localstore.go

@@ -19,6 +19,7 @@ package storage
 import (
 	"encoding/binary"
 	"fmt"
+	"path/filepath"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -30,50 +31,59 @@ var (
 	dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil)
 )
 
-type StoreParams struct {
+type LocalStoreParams struct {
-	ChunkDbPath   string
+	*StoreParams
-	DbCapacity    uint64
+	ChunkDbPath string
-	CacheCapacity uint
+	Validators  []ChunkValidator `toml:"-"`
-	Radius        int
 }
 
-//create params with default values
+func NewDefaultLocalStoreParams() *LocalStoreParams {
-func NewDefaultStoreParams() (self *StoreParams) {
+	return &LocalStoreParams{
-	return &StoreParams{
+		StoreParams: NewStoreParams(0, nil, nil),
-		DbCapacity:    defaultDbCapacity,
+	}
-		CacheCapacity: defaultCacheCapacity,
+}
+
+//this can only finally be set after all config options (file, cmd line, env vars)
+//have been evaluated
+func (self *LocalStoreParams) Init(path string) {
+	if self.ChunkDbPath == "" {
+		self.ChunkDbPath = filepath.Join(path, "chunks")
 	}
 }
 
 // LocalStore is a combination of inmemory db over a disk persisted db
 // implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
 type LocalStore struct {
-	memStore *MemStore
+	Validators []ChunkValidator
-	DbStore  *LDBStore
+	memStore   *MemStore
-	mu       sync.Mutex
+	DbStore    *LDBStore
+	mu         sync.Mutex
 }
 
 // This constructor uses MemStore and DbStore as components
-func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) {
+func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error) {
-	dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore)
+	ldbparams := NewLDBStoreParams(params.ChunkDbPath, params.DbCapacity, params.Hash, params.BaseKey)
+	dbStore, err := NewMockDbStore(ldbparams, mockStore)
 	if err != nil {
 		return nil, err
 	}
 	return &LocalStore{
-		memStore: NewMemStore(dbStore, params.CacheCapacity),
+		memStore:   NewMemStore(params.StoreParams, dbStore),
-		DbStore:  dbStore,
+		DbStore:    dbStore,
+		Validators: params.Validators,
 	}, nil
 }
 
-func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) {
+func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) {
-	hasher := MakeHashFunc("SHA3")
+	ldbparams := NewLDBStoreParams(params.ChunkDbPath, params.DbCapacity, params.Hash, params.BaseKey)
-	dbStore, err := NewLDBStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
+	dbStore, err := NewLDBStore(ldbparams)
 	if err != nil {
 		return nil, err
 	}
 	localStore := &LocalStore{
-		memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
+		memStore:   NewMemStore(params.StoreParams, dbStore),
-		DbStore:  dbStore,
+		DbStore:    dbStore,
+		Validators: params.Validators,
 	}
 	return localStore, nil
 }
@@ -82,9 +92,18 @@ func (self *LocalStore) CacheCounter() uint64 {
 	return uint64(self.memStore.Counter())
 }
 
-// LocalStore is itself a chunk store
-// unsafe, in that the data is not integrity checked
 func (self *LocalStore) Put(chunk *Chunk) {
+	valid := true
+	for _, v := range self.Validators {
+		if valid = v.Validate(chunk.Key, chunk.SData); valid {
+			break
+		}
+	}
+	if !valid {
+		chunk.SetErrored(ErrChunkInvalid)
+		chunk.dbStoredC <- false
+		return
+	}
 	log.Trace("localstore.put", "key", chunk.Key)
 	self.mu.Lock()
 	defer self.mu.Unlock()
@@ -143,11 +162,11 @@ func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool)
 
 	var err error
 	chunk, err = self.get(key)
-	if err == nil && !chunk.GetErrored() {
+	if err == nil && chunk.GetErrored() == nil {
 		log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key))
 		return chunk, false
 	}
-	if err == ErrFetching && !chunk.GetErrored() {
+	if err == ErrFetching && chunk.GetErrored() == nil {
 		log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC))
 		return chunk, false
 	}

swarm/storage/localstore_test.go

@@ -0,0 +1,153 @@
+package storage
+
+import (
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/contracts/ens"
+)
+
+var (
+	hashfunc = MakeHashFunc("SHA3")
+)
+
+// convenience generator for unique chunks
+type randomChunkGenerator struct {
+	data   []byte
+	hasher SwarmHash
+}
+
+func newRandomChunkGenerator(stem []byte) *randomChunkGenerator {
+	gen := &randomChunkGenerator{
+		data:   make([]byte, 8),
+		hasher: hashfunc(),
+	}
+	gen.data = append(gen.data, stem...)
+	return gen
+}
+
+func (self *randomChunkGenerator) newChunk() *Chunk {
+	self.hasher.Reset()
+	self.hasher.Write(self.data)
+	chunk := NewChunk(self.hasher.Sum(nil), nil)
+	chunk.SData = make([]byte, len(self.data))
+	copy(chunk.SData, self.data)
+	self.data[0]++
+	return chunk
+}
+
+// put to localstore and wait for stored channel
+// does not check delivery error state
+func putChunks(store *LocalStore, chunks ...*Chunk) {
+	wg := sync.WaitGroup{}
+	wg.Add(len(chunks))
+	go func() {
+		for _, c := range chunks {
+			<-c.dbStoredC
+			wg.Done()
+		}
+	}()
+	for _, c := range chunks {
+		go store.Put(c)
+	}
+	wg.Wait()
+}
+
+// tests that the content address validator correctly checks the data
+// tests that resource update chunks are passed through content address validator
+// the test checking the resouce update validator internal correctness is found in resource_test.go
+func TestValidator(t *testing.T) {
+	bogusData := []byte("00000000bar")
+
+	// set up localstore
+	datadir, err := ioutil.TempDir("", "storage-testvalidator")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(datadir)
+
+	params := NewDefaultLocalStoreParams()
+	params.Init(datadir)
+	store, err := NewLocalStore(params, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// check puts with no validators, both succeed
+	gen := newRandomChunkGenerator([]byte("foo"))
+	goodChunk := gen.newChunk()
+	badChunk := gen.newChunk()
+	badChunk.SData = bogusData
+
+	putChunks(store, goodChunk, badChunk)
+	if err := goodChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on good content address chunk in spite of no validation, but got: %s", err)
+	}
+	if err := badChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err)
+	}
+
+	// add content address validator and check puts
+	// bad should fail, good should pass
+	store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
+	goodChunk = gen.newChunk()
+	badChunk = gen.newChunk()
+	badChunk.SData = bogusData
+
+	putChunks(store, goodChunk, badChunk)
+	if err := goodChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err)
+	}
+	if err := badChunk.GetErrored(); err == nil {
+		t.Fatal("expected error on bad content address chunk with content address validator only, but got nil")
+	}
+
+	// append resource validator to validators and check puts
+	// bad should fail, good should pass, resource should pass
+	rhParams := &ResourceHandlerParams{}
+	rh, err := NewResourceHandler(rhParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+	store.Validators = append(store.Validators, rh)
+
+	goodChunk = gen.newChunk()
+	key := rh.resourceHash(42, 1, ens.EnsNode("xyzzy.eth"))
+	data := []byte("bar")
+	uglyChunk := newUpdateChunk(key, nil, 42, 1, "xyzzy.eth", data, len(data))
+
+	putChunks(store, goodChunk, badChunk, uglyChunk)
+	if err := goodChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on good content address chunk with both validators, but got: %s", err)
+	}
+	if err := badChunk.GetErrored(); err == nil {
+		t.Fatal("expected error on bad chunk address with both validators, but got nil")
+	}
+	if err := uglyChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on resource update chunk with both validators, but got: %s", err)
+	}
+
+	// (redundant check)
+	// use only resource validator, and check puts
+	// bad should fail, good should fail, resource should pass
+	store.Validators[0] = store.Validators[1]
+	store.Validators = store.Validators[:1]
+
+	goodChunk = gen.newChunk()
+	key = rh.resourceHash(42, 2, ens.EnsNode("xyzzy.eth"))
+	data = []byte("baz")
+	uglyChunk = newUpdateChunk(key, nil, 42, 2, "xyzzy.eth", data, len(data))
+
+	putChunks(store, goodChunk, badChunk, uglyChunk)
+	if goodChunk.GetErrored() == nil {
+		t.Fatal("expected error on good content address chunk with resource validator only, but got nil")
+	}
+	if badChunk.GetErrored() == nil {
+		t.Fatal("expected error on bad content address chunk with resource validator only, but got nil")
+	}
+	if err := uglyChunk.GetErrored(); err != nil {
+		t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err)
+	}
+}

swarm/storage/memstore.go

@@ -62,7 +62,9 @@ a hash prefix subtree containing subtrees or one storage entry (but never both)
   (access[] is a binary tree inside the multi-bit leveled hash tree)
 */
 
-func NewMemStore(d *LDBStore, capacity uint) (m *MemStore) {
+func NewMemStore(params *StoreParams, d *LDBStore) (m *MemStore) {
+
+	capacity := params.CacheCapacity
 	m = &MemStore{}
 	m.memtree = newMemTree(memTreeFLW, nil, 0)
 	m.ldbStore = d
@@ -260,7 +262,6 @@ func (s *MemStore) removeOldest() {
 	defer metrics.GetOrRegisterResettingTimer("memstore.purge", metrics.DefaultRegistry).UpdateSince(time.Now())
 
 	node := s.memtree
-	log.Warn("purge memstore")
 	for node.entry == nil {
 
 		aidx := uint(0)

swarm/storage/memstore_test.go

@@ -19,7 +19,8 @@ package storage
 import "testing"
 
 func newTestMemStore() *MemStore {
-	return NewMemStore(nil, defaultCacheCapacity)
+	storeparams := NewStoreParams(defaultCacheCapacity, nil, nil)
+	return NewMemStore(storeparams, nil)
 }
 
 func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) {

swarm/storage/netstore.go

@@ -17,7 +17,6 @@
 package storage
 
 import (
-	"path/filepath"
 	"time"
 )
 
@@ -57,7 +56,7 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
 			err := self.retrieve(chunk)
 			if err != nil {
 				// mark chunk request as failed so that we can retry it later
-				chunk.SetErrored(true)
+				chunk.SetErrored(ErrChunkUnavailable)
 				return nil, err
 			}
 		}
@@ -69,22 +68,14 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
 	select {
 	case <-t.C:
 		// mark chunk request as failed so that we can retry
-		chunk.SetErrored(true)
+		chunk.SetErrored(ErrChunkNotFound)
 		return nil, ErrChunkNotFound
 	case <-chunk.ReqC:
 	}
-	chunk.SetErrored(false)
+	chunk.SetErrored(nil)
 	return chunk, nil
 }
 
-//this can only finally be set after all config options (file, cmd line, env vars)
-//have been evaluated
-func (self *StoreParams) Init(path string) {
-	if self.ChunkDbPath == "" {
-		self.ChunkDbPath = filepath.Join(path, "chunks")
-	}
-}
-
 // Put is the entrypoint for local store requests coming from storeLoop
 func (self *NetStore) Put(chunk *Chunk) {
 	self.localStore.Put(chunk)

swarm/storage/netstore_test.go

@@ -80,8 +80,10 @@ func TestNetstoreFailedRequest(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	params := NewDefaultLocalStoreParams()
-	localStore, err := NewTestLocalStoreForAddr(datadir, addr.Over())
+	params.Init(datadir)
+	params.BaseKey = addr.Over()
+	localStore, err := NewTestLocalStoreForAddr(params)
 	if err != nil {
 		t.Fatal(err)
 	}

swarm/storage/resource.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"bytes"
 	"context"
 	"encoding/binary"
 	"errors"
@@ -13,6 +14,7 @@ import (
 	"golang.org/x/net/idna"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contracts/ens"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
@@ -20,11 +22,12 @@ import (
 
 const (
 	signatureLength     = 65
-	indexSize           = 16
+	indexSize           = 18
 	DbDirName           = "resource"
 	chunkSize           = 4096 // temporary until we implement DPA in the resourcehandler
 	defaultStoreTimeout = 4000 * time.Millisecond
 	hasherCount         = 8
+	resourceHash        = SHA3Hash
 )
 
 type ResourceError struct {
@@ -61,10 +64,6 @@ type ResourceLookupParams struct {
 	Max   uint32
 }
 
-type SignFunc func(common.Hash) (Signature, error)
-
-type nameHashFunc func(string) common.Hash
-
 // Encapsulates an specific resource update. When synced it contains the most recent
 // version of the resource update data.
 type resource struct {
@@ -88,15 +87,6 @@ func (self *resource) NameHash() common.Hash {
 	return self.nameHash
 }
 
-// Implement to activate validation of resource updates
-// Specifically signing data and verification of signatures
-type ResourceValidator interface {
-	hashSize() int
-	checkAccess(string, common.Address) (bool, error)
-	NameHash(string) common.Hash         // nameHashFunc
-	sign(common.Hash) (Signature, error) // SignFunc
-}
-
 type headerGetter interface {
 	HeaderByNumber(context.Context, string, *big.Int) (*types.Header, error)
 }
@@ -146,90 +136,123 @@ type headerGetter interface {
 // headerlength|period|version|identifier|data
 //
 // if a validator is active, the chunk data is:
-// sign(resourcedata)|resourcedata
+// resourcedata|sign(resourcedata)
 // otherwise, the chunk data is the same as the resourcedata
 //
 // headerlength is a 16 bit value containing the byte length of period|version|name
-// period and version are both 32 bit values. name can have arbitrary length
-//
-// NOTE: the following is yet to be implemented
-// The resource update chunks will be stored in the swarm, but receive special
-// treatment as their keys do not validate as hashes of their data. They are also
-// stored using a separate store, and forwarding/syncing protocols carry per-chunk
-// flags to tell whether the chunk can be validated or not; if not it is to be
-// treated as a resource update chunk.
 //
 // TODO: Include modtime in chunk data + signature
 type ResourceHandler struct {
-	ChunkStore
+	chunkStore      ChunkStore
-	validator       ResourceValidator
+	HashSize        int
+	signer          ResourceSigner
 	ethClient       headerGetter
+	ensClient       *ens.ENS
 	resources       map[string]*resource
 	hashPool        sync.Pool
 	resourceLock    sync.RWMutex
-	nameHash        nameHashFunc
 	storeTimeout    time.Duration
 	queryMaxPeriods *ResourceLookupParams
 }
 
 type ResourceHandlerParams struct {
-	Validator       ResourceValidator
 	QueryMaxPeriods *ResourceLookupParams
+	Signer          ResourceSigner
+	EthClient       headerGetter
+	EnsClient       *ens.ENS
 }
 
 // Create or open resource update chunk store
-func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, params *ResourceHandlerParams) (*ResourceHandler, error) {
+func NewResourceHandler(params *ResourceHandlerParams) (*ResourceHandler, error) {
 	if params.QueryMaxPeriods == nil {
 		params.QueryMaxPeriods = &ResourceLookupParams{
 			Limit: false,
 		}
 	}
 	rh := &ResourceHandler{
-		ChunkStore:   chunkStore,
+		ethClient:    params.EthClient,
-		ethClient:    ethClient,
+		ensClient:    params.EnsClient,
 		resources:    make(map[string]*resource),
-		validator:    params.Validator,
 		storeTimeout: defaultStoreTimeout,
+		signer:       params.Signer,
 		hashPool: sync.Pool{
 			New: func() interface{} {
-				return MakeHashFunc(SHA3Hash)()
+				return MakeHashFunc(resourceHash)()
 			},
 		},
 		queryMaxPeriods: params.QueryMaxPeriods,
 	}
 
-	if rh.validator != nil {
-		rh.nameHash = rh.validator.NameHash
-	} else {
-		rh.nameHash = func(name string) common.Hash {
-			hasher := rh.hashPool.Get().(SwarmHash)
-			defer rh.hashPool.Put(hasher)
-			hasher.Reset()
-			hasher.Write([]byte(name))
-			hashval := common.BytesToHash(hasher.Sum(nil))
-			log.Debug("generic namehasher", "name", name, "hash", hashval)
-			return hashval
-		}
-	}
-
 	for i := 0; i < hasherCount; i++ {
-		hashfunc := MakeHashFunc(SHA3Hash)()
+		hashfunc := MakeHashFunc(resourceHash)()
+		if rh.HashSize == 0 {
+			rh.HashSize = hashfunc.Size()
+		}
 		rh.hashPool.Put(hashfunc)
 	}
 
 	return rh, nil
 }
 
-func (self *ResourceHandler) IsValidated() bool {
+// Sets the store backend for resource updates
-	return self.validator == nil
+func (self *ResourceHandler) SetStore(store ChunkStore) {
+	self.chunkStore = store
 }
 
-func (self *ResourceHandler) HashSize() int {
+// Chunk Validation method (matches ChunkValidatorFunc signature)
-	return self.validator.hashSize()
+//
+// If resource update, owner is checked against ENS record of resource name inferred from chunk data
+// If parsed signature is nil, validates automatically
+// If not resource update, it validates are root chunk if length is indexSize and first two bytes are 0
+func (self *ResourceHandler) Validate(key Key, data []byte) bool {
+	signature, period, version, name, parseddata, err := self.parseUpdate(data)
+	if err != nil {
+		if len(data) == indexSize {
+			if bytes.Equal(data[:2], []byte{0, 0}) {
+				return true
+			}
+		}
+		return false
+	} else if signature == nil {
+		if !bytes.Equal(self.resourceHash(period, version, ens.EnsNode(name)), key) {
+			return false
+		}
+		return true
+	}
+	digest := self.keyDataHash(key, parseddata)
+	addr, err := getAddressFromDataSig(digest, *signature)
+	if err != nil {
+		return false
+	}
+	ok, _ := self.checkAccess(name, addr)
+	return ok
+}
+
+// If no ens client is supplied, resource updates are not validated
+func (self *ResourceHandler) IsValidated() bool {
+	return self.ensClient != nil
+}
+
+// Create the resource update digest used in signatures
+func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash {
+	hasher := self.hashPool.Get().(SwarmHash)
+	defer self.hashPool.Put(hasher)
+	hasher.Reset()
+	hasher.Write(key[:])
+	hasher.Write(data)
+	return common.BytesToHash(hasher.Sum(nil))
+}
+
+// Checks if current address matches owner address of ENS
+func (self *ResourceHandler) checkAccess(name string, address common.Address) (bool, error) {
+	if self.ensClient == nil {
+		return true, nil
+	}
+	owneraddress, err := self.ensClient.Owner(ens.EnsNode(name))
+	return (address == owneraddress), err
 }
 
 // get data from current resource
-
 func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) {
 	rsrc := self.getResource(name)
 	if rsrc == nil || !rsrc.isSynced() {
@@ -278,10 +301,10 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
 		return nil, NewResourceError(ErrInvalidValue, fmt.Sprintf("Invalid name: '%s'", name))
 	}
 
-	nameHash := self.nameHash(name)
+	nameHash := ens.EnsNode(name)
 
-	if self.validator != nil {
+	if self.signer != nil {
-		signature, err := self.validator.sign(nameHash)
+		signature, err := self.signer.Sign(nameHash)
 		if err != nil {
 			return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err))
 		}
@@ -289,7 +312,7 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
 		if err != nil {
 			return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Retrieve address from signature fail: %v", err))
 		}
-		ok, err := self.validator.checkAccess(name, addr)
+		ok, err := self.checkAccess(name, addr)
 		if err != nil {
 			return nil, err
 		} else if !ok {
@@ -305,15 +328,17 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
 
 	// chunk with key equal to namehash points to data of first blockheight + update frequency
 	// from this we know from what blockheight we should look for updates, and how often
-	chunk := NewChunk(Key(nameHash.Bytes()), nil)
+	key := Key(nameHash.Bytes())
+	chunk := NewChunk(key, nil)
 	chunk.SData = make([]byte, indexSize)
 
+	// root block has first two bytes both, which distinguishes from update bytes
 	val := make([]byte, 8)
 	binary.LittleEndian.PutUint64(val, currentblock)
-	copy(chunk.SData[:8], val)
+	copy(chunk.SData[2:10], val)
 	binary.LittleEndian.PutUint64(val, frequency)
-	copy(chunk.SData[8:], val)
+	copy(chunk.SData[10:], val)
-	self.Put(chunk)
+	self.chunkStore.Put(chunk)
 	log.Debug("new resource", "name", name, "key", nameHash, "startBlock", currentblock, "frequency", frequency)
 
 	rsrc := &resource{
@@ -336,12 +361,8 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
 // root chunk.
 // It is the callers responsibility to make sure that this chunk exists (if the resource
 // update root data was retrieved externally, it typically doesn't)
-//
-// If maxPeriod is -1, the default QueryMaxPeriod from ResourceHandlerParams will be used
-// if maxPeriod is 0, there will be no limit on period hops
-// if maxPeriod > 0, the given value will be the limit of period hops
 func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
-	return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh, maxLookup)
+	return self.LookupVersion(ctx, ens.EnsNode(name), name, period, version, refresh, maxLookup)
 }
 
 func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@@ -361,7 +382,7 @@ func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.
 //
 // See also (*ResourceHandler).LookupVersion
 func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
-	return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh, maxLookup)
+	return self.LookupHistorical(ctx, ens.EnsNode(name), name, period, refresh, maxLookup)
 }
 
 func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@@ -383,7 +404,7 @@ func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash comm
 //
 // See also (*ResourceHandler).LookupHistorical
 func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
-	return self.LookupLatest(ctx, self.nameHash(name), name, refresh, maxLookup)
+	return self.LookupLatest(ctx, ens.EnsNode(name), name, refresh, maxLookup)
 }
 
 func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@@ -404,6 +425,10 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H
 // base code for public lookup methods
 func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
 
+	if self.chunkStore == nil {
+		return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before performing lookups")
+	}
+
 	if period == 0 {
 		return nil, NewResourceError(ErrInvalidValue, "period must be >0")
 	}
@@ -426,7 +451,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
 			return nil, NewResourceError(ErrPeriodDepth, fmt.Sprintf("Lookup exceeded max period hops (%d)", maxLookup.Max))
 		}
 		key := self.resourceHash(period, version, rsrc.nameHash)
-		chunk, err := self.Get(key)
+		chunk, err := self.chunkStore.Get(key)
 		if err == nil {
 			if specificversion {
 				return self.updateResourceIndex(rsrc, chunk)
@@ -436,7 +461,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
 			for {
 				newversion := version + 1
 				key := self.resourceHash(period, newversion, rsrc.nameHash)
-				newchunk, err := self.Get(key)
+				newchunk, err := self.chunkStore.Get(key)
 				if err != nil {
 					return self.updateResourceIndex(rsrc, chunk)
 				}
@@ -472,7 +497,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref
 		rsrc.nameHash = nameHash
 
 		// get the root info chunk and update the cached value
-		chunk, err := self.Get(Key(rsrc.nameHash[:]))
+		chunk, err := self.chunkStore.Get(Key(rsrc.nameHash[:]))
 		if err != nil {
 			return nil, err
 		}
@@ -481,8 +506,8 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref
 		if len(chunk.SData) != indexSize {
 			return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize))
 		}
-		rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8])
+		rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[2:10])
-		rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:])
+		rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[10:])
 	} else {
 		rsrc.name = self.resources[name].name
 		rsrc.nameHash = self.resources[name].nameHash
@@ -501,8 +526,8 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) (
 		return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name))
 	}
 	log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version)
-	// only check signature if validator is present
+	// check signature (if signer algorithm is present)
-	if self.validator != nil {
+	if signature != nil {
 		digest := self.keyDataHash(chunk.Key, data)
 		_, err = getAddressFromDataSig(digest, *signature)
 		if err != nil {
@@ -525,11 +550,16 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) (
 // retrieve update metadata from chunk data
 // mirrors newUpdateChunk()
 func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) {
+	// 14 = header + one byte of name + one byte of data
+	if len(chunkdata) < 14 {
+		return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, "chunk less than 13 bytes cannot be a resource update chunk")
+	}
 	cursor := 0
 	headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2])
 	cursor += 2
 	datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2])
-	if int(headerlength+datalength+4) > len(chunkdata) {
+	exclsignlength := int(headerlength + datalength + 4)
+	if exclsignlength > len(chunkdata) || exclsignlength < 14 {
 		return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)))
 	}
 	var period uint32
@@ -560,10 +590,13 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32,
 
 	// omit signatures if we have no validator
 	var signature *Signature
-	if self.validator != nil {
+	cursor += intdatalength
-		cursor += intdatalength
+	if self.signer != nil {
-		signature = &Signature{}
+		sigdata := chunkdata[cursor : cursor+signatureLength]
-		copy(signature[:], chunkdata[cursor:cursor+signatureLength])
+		if len(sigdata) > 0 {
+			signature = &Signature{}
+			copy(signature[:], sigdata)
+		}
 	}
 
 	return signature, period, version, name, data, nil
@@ -589,8 +622,11 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt
 
 func (self *ResourceHandler) update(ctx context.Context, name string, data []byte, multihash bool) (Key, error) {
 
+	if self.chunkStore == nil {
+		return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before updating")
+	}
 	var signaturelength int
-	if self.validator != nil {
+	if self.signer != nil {
 		signaturelength = signatureLength
 	}
 
@@ -628,10 +664,10 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
 	key := self.resourceHash(nextperiod, version, rsrc.nameHash)
 
 	var signature *Signature
-	if self.validator != nil {
+	if self.signer != nil {
 		// sign the data hash with the key
 		digest := self.keyDataHash(key, data)
-		sig, err := self.validator.sign(digest)
+		sig, err := self.signer.Sign(digest)
 		if err != nil {
 			return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err))
 		}
@@ -642,13 +678,14 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
 		if err != nil {
 			return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Invalid data/signature: %v", err))
 		}
-
+		if self.signer != nil {
-		// check if the signer has access to update
+			// check if the signer has access to update
-		ok, err := self.validator.checkAccess(name, addr)
+			ok, err := self.checkAccess(name, addr)
-		if err != nil {
+			if err != nil {
-			return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err))
+				return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err))
-		} else if !ok {
+			} else if !ok {
-			return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name))
+				return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name))
+			}
 		}
 	}
 
@@ -659,7 +696,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
 	chunk := newUpdateChunk(key, signature, nextperiod, version, name, data, datalength)
 
 	// send the chunk
-	self.Put(chunk)
+	self.chunkStore.Put(chunk)
 	timeout := time.NewTimer(self.storeTimeout)
 	select {
 	case <-chunk.dbStoredC:
@@ -679,7 +716,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
 // Closes the datastore.
 // Always call this at shutdown to avoid data corruption.
 func (self *ResourceHandler) Close() {
-	self.ChunkStore.Close()
+	self.chunkStore.Close()
 }
 
 func (self *ResourceHandler) getBlock(ctx context.Context, name string) (uint64, error) {
@@ -789,52 +826,6 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32
 	return chunk
 }
 
-// \TODO chunkSize is a workaround until the ChunkStore interface exports a method to get the chunk size directly
-type resourceChunkStore struct {
-	localStore ChunkStore
-	netStore   ChunkStore
-	chunkSize  int64
-}
-
-func NewResourceChunkStore(localStore *LocalStore, request func(*Chunk) error) ChunkStore {
-	return &resourceChunkStore{
-		localStore: localStore,
-		netStore:   NewNetStore(localStore, request),
-	}
-}
-
-func (r *resourceChunkStore) Get(key Key) (*Chunk, error) {
-	chunk, err := r.netStore.Get(key)
-	if err != nil {
-		return nil, err
-	}
-	// if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing.
-	// sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue
-	// that caused the failure or that the chunk doesn't exist.
-	if chunk.ReqC == nil {
-		return chunk, nil
-	}
-	t := time.NewTimer(time.Second * 1)
-	select {
-	case <-t.C:
-		log.Trace("Timeout on resource chunk store")
-		return nil, fmt.Errorf("timeout")
-	case <-chunk.C:
-		log.Trace("Received resource update chunk")
-	}
-	return chunk, nil
-}
-
-func (r *resourceChunkStore) Put(chunk *Chunk) {
-	r.netStore.Put(chunk)
-	chunk.WaitToStore()
-}
-
-func (r *resourceChunkStore) Close() {
-	r.netStore.Close()
-	r.localStore.Close()
-}
-
 func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 {
 	blockdiff := current - start
 	period := blockdiff / frequency
@@ -857,16 +848,6 @@ func isSafeName(name string) bool {
 	return validname == name
 }
 
-// convenience for creating signature hashes of update data
-func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash {
-	hasher := self.hashPool.Get().(SwarmHash)
-	defer self.hashPool.Put(hasher)
-	hasher.Reset()
-	hasher.Write(key[:])
-	hasher.Write(data)
-	return common.BytesToHash(hasher.Sum(nil))
-}
-
 // if first byte is the start of a multihash this function will try to parse it
 // if successful it returns the length of multihash data, 0 otherwise
 func isMultihash(data []byte) int {
@@ -892,29 +873,20 @@ func isMultihash(data []byte) int {
 	return cursor + inthashlength
 }
 
-// TODO: this should not be part of production code, but currently swarm/testutil/http.go needs it
+// TODO: this should not be exposed, but swarm/testutil/http.go needs it
-func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator, maxLimit *ResourceLookupParams) (*ResourceHandler, error) {
+func NewTestResourceHandler(datadir string, params *ResourceHandlerParams) (*ResourceHandler, error) {
 	path := filepath.Join(datadir, DbDirName)
-	basekey := make([]byte, 32)
+	rh, err := NewResourceHandler(params)
-	hasher := MakeHashFunc(SHA3Hash)
-	dbStore, err := NewLDBStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
-	dbStore.SetTrusted()
 	if err != nil {
 		return nil, err
 	}
-	localStore := &LocalStore{
+	localstoreparams := NewDefaultLocalStoreParams()
-		memStore: NewMemStore(dbStore, singletonSwarmDbCapacity),
+	localstoreparams.Init(path)
-		DbStore:  dbStore,
+	localStore, err := NewLocalStore(localstoreparams, nil)
+	if err != nil {
+		return nil, err
 	}
-	resourceChunkStore := NewResourceChunkStore(localStore, nil)
+	localStore.Validators = append(localStore.Validators, rh)
-	if maxLimit == nil {
+	rh.SetStore(localStore)
-		maxLimit = &ResourceLookupParams{
+	return rh, nil
-			Limit: false,
-		}
-	}
-	params := &ResourceHandlerParams{
-		Validator:       validator,
-		QueryMaxPeriods: maxLimit,
-	}
-	return NewResourceHandler(hasher, resourceChunkStore, ethClient, params)
 }

swarm/storage/resource_ens.go

@@ -1,52 +0,0 @@
-package storage
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/contracts/ens"
-)
-
-type baseValidator struct {
-	signFunc SignFunc
-	hashsize int
-}
-
-func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) {
-	if b.signFunc == nil {
-		return signature, errors.New("No signature function")
-	}
-	return b.signFunc(datahash)
-}
-
-func (b *baseValidator) hashSize() int {
-	return b.hashsize
-}
-
-type OwnerValidator interface {
-	ValidateOwner(name string, address common.Address) (bool, error)
-}
-
-// ENS validation of mutable resource owners
-type ENSValidator struct {
-	*baseValidator
-	api OwnerValidator
-}
-
-func NewENSValidator(contractaddress common.Address, ownerValidator OwnerValidator, signFunc SignFunc) *ENSValidator {
-	return &ENSValidator{
-		baseValidator: &baseValidator{
-			signFunc: signFunc,
-			hashsize: common.HashLength,
-		},
-		api: ownerValidator,
-	}
-}
-
-func (self *ENSValidator) checkAccess(name string, address common.Address) (bool, error) {
-	return self.api.ValidateOwner(name, address)
-}
-
-func (self *ENSValidator) NameHash(name string) common.Hash {
-	return ens.EnsNode(name)
-}

swarm/storage/resource_sign.go

@@ -7,14 +7,20 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 )
 
-// matches the SignFunc type
+// Signs resource updates
-func NewGenericResourceSigner(privKey *ecdsa.PrivateKey) SignFunc {
+type ResourceSigner interface {
-	return func(data common.Hash) (signature Signature, err error) {
+	Sign(common.Hash) (Signature, error)
-		signaturebytes, err := crypto.Sign(data.Bytes(), privKey)
+}
-		if err != nil {
+
-			return
+type GenericResourceSigner struct {
-		}
+	PrivKey *ecdsa.PrivateKey
-		copy(signature[:], signaturebytes)
+}
+
+func (self *GenericResourceSigner) Sign(data common.Hash) (signature Signature, err error) {
+	signaturebytes, err := crypto.Sign(data.Bytes(), self.PrivKey)
+	if err != nil {
 		return
 	}
+	copy(signature[:], signaturebytes)
+	return
 }

swarm/storage/resource_test.go

@@ -3,7 +3,6 @@ package storage
 import (
 	"bytes"
 	"context"
-	"crypto/ecdsa"
 	"crypto/rand"
 	"encoding/binary"
 	"flag"
@@ -83,14 +82,14 @@ func TestResourceReverse(t *testing.T) {
 	}
 
 	// set up rpc and create resourcehandler
-	rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent))
+	rh, _, teardownTest, err := setupTest(nil, nil, signer)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer teardownTest()
 
 	// generate a hash for block 4200 version 1
-	key := rh.resourceHash(period, version, rh.nameHash(safeName))
+	key := rh.resourceHash(period, version, ens.EnsNode(safeName))
 
 	// generate some bogus data for the chunk and sign it
 	data := make([]byte, 8)
@@ -101,7 +100,7 @@ func TestResourceReverse(t *testing.T) {
 	testHasher.Reset()
 	testHasher.Write(data)
 	digest := rh.keyDataHash(key, data)
-	sig, err := rh.validator.sign(digest)
+	sig, err := rh.signer.Sign(digest)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -118,7 +117,7 @@ func TestResourceReverse(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Retrieve address from signature fail: %v", err)
 	}
-	originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey)
+	originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
 
 	// check that the metadata retrieved from the chunk matches what we gave it
 	if recoveredaddress != originaladdress {
@@ -149,7 +148,7 @@ func TestResourceHandler(t *testing.T) {
 	backend := &fakeBackend{
 		blocknumber: int64(startBlock),
 	}
-	rh, datadir, _, teardownTest, err := setupTest(backend, nil)
+	rh, datadir, teardownTest, err := setupTest(backend, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -164,15 +163,15 @@ func TestResourceHandler(t *testing.T) {
 	}
 
 	// check that the new resource is stored correctly
-	namehash := rh.nameHash(safeName)
+	namehash := ens.EnsNode(safeName)
-	chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:]))
+	chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(Key(namehash[:]))
 	if err != nil {
 		t.Fatal(err)
 	} else if len(chunk.SData) < 16 {
 		t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData))
 	}
-	startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8])
+	startblocknumber := binary.LittleEndian.Uint64(chunk.SData[2:10])
-	chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:])
+	chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[10:])
 	if startblocknumber != uint64(backend.blocknumber) {
 		t.Fatalf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber)
 	}
@@ -219,10 +218,14 @@ func TestResourceHandler(t *testing.T) {
 	// it will match on second iteration startblocknumber + (resourceFrequency * 3)
 	fwdBlocks(int(resourceFrequency*2)-1, backend)
 
-	lookupParams := &ResourceLookupParams{
+	rhparams := &ResourceHandlerParams{
-		Limit: false,
+		QueryMaxPeriods: &ResourceLookupParams{
+			Limit: false,
+		},
+		Signer:    nil,
+		EthClient: rh.ethClient,
 	}
-	rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil, lookupParams)
+	rh2, err := NewTestResourceHandler(datadir, rhparams)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -244,7 +247,7 @@ func TestResourceHandler(t *testing.T) {
 	log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
 
 	// specific block, latest version
-	rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, lookupParams)
+	rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, rh2.queryMaxPeriods)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -255,7 +258,7 @@ func TestResourceHandler(t *testing.T) {
 	log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
 
 	// specific block, specific version
-	rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, lookupParams)
+	rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, rh2.queryMaxPeriods)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -267,6 +270,65 @@ func TestResourceHandler(t *testing.T) {
 
 }
 
+// create ENS enabled resource update, with and without valid owner
+func TestResourceENSOwner(t *testing.T) {
+
+	// signer containing private key
+	signer, err := newTestSigner()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ens address and transact options
+	addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
+	transactOpts := bind.NewKeyedTransactor(signer.PrivKey)
+
+	// set up ENS sim
+	domainparts := strings.Split(safeName, ".")
+	contractAddr, contractbackend, err := setupENS(addr, transactOpts, domainparts[0], domainparts[1])
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ensClient, err := ens.NewENS(transactOpts, contractAddr, contractbackend)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// set up rpc and create resourcehandler with ENS sim backend
+	rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer teardownTest()
+
+	// create new resource when we are owner = ok
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	_, err = rh.NewResource(ctx, safeName, resourceFrequency)
+	if err != nil {
+		t.Fatalf("Create resource fail: %v", err)
+	}
+
+	data := []byte("foo")
+	// update resource when we are owner = ok
+	_, err = rh.Update(ctx, safeName, data)
+	if err != nil {
+		t.Fatalf("Update resource fail: %v", err)
+	}
+
+	// update resource when we are not owner = !ok
+	signertwo, err := newTestSigner()
+	if err != nil {
+		t.Fatal(err)
+	}
+	rh.signer = signertwo
+	_, err = rh.Update(ctx, safeName, data)
+	if err == nil {
+		t.Fatalf("Expected resource update fail due to owner mismatch")
+	}
+}
+
 func TestResourceMultihash(t *testing.T) {
 
 	// signer containing private key
@@ -274,7 +336,6 @@ func TestResourceMultihash(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	validator := newTestValidator(signer.signContent)
 
 	// make fake backend, set up rpc and create resourcehandler
 	backend := &fakeBackend{
@@ -282,7 +343,7 @@ func TestResourceMultihash(t *testing.T) {
 	}
 
 	// set up rpc and create resourcehandler
-	rh, datadir, _, teardownTest, err := setupTest(backend, nil)
+	rh, datadir, teardownTest, err := setupTest(backend, nil, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -298,7 +359,7 @@ func TestResourceMultihash(t *testing.T) {
 
 	// we're naïvely assuming keccak256 for swarm hashes
 	// if it ever changes this test should also change
-	swarmhashbytes := rh.nameHash("foo")
+	swarmhashbytes := ens.EnsNode("foo")
 	swarmhashmulti, err := multihash.Encode(swarmhashbytes.Bytes(), multihash.KECCAK_256)
 	if err != nil {
 		t.Fatal(err)
@@ -352,8 +413,16 @@ func TestResourceMultihash(t *testing.T) {
 	}
 	rh.Close()
 
+	rhparams := &ResourceHandlerParams{
+		QueryMaxPeriods: &ResourceLookupParams{
+			Limit: false,
+		},
+		Signer:    signer,
+		EthClient: rh.ethClient,
+		EnsClient: rh.ensClient,
+	}
 	// test with signed data
-	rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator, nil)
+	rh2, err := NewTestResourceHandler(datadir, rhparams)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -394,9 +463,7 @@ func TestResourceMultihash(t *testing.T) {
 	}
 }
 
-// create ENS enabled resource update, with and without valid owner
+func TestResourceChunkValidator(t *testing.T) {
-func TestResourceENSOwner(t *testing.T) {
-
 	// signer containing private key
 	signer, err := newTestSigner()
 	if err != nil {
@@ -404,8 +471,8 @@ func TestResourceENSOwner(t *testing.T) {
 	}
 
 	// ens address and transact options
-	addr := crypto.PubkeyToAddress(signer.privKey.PublicKey)
+	addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
-	transactOpts := bind.NewKeyedTransactor(signer.privKey)
+	transactOpts := bind.NewKeyedTransactor(signer.PrivKey)
 
 	// set up ENS sim
 	domainparts := strings.Split(safeName, ".")
@@ -418,10 +485,9 @@ func TestResourceENSOwner(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	validator := NewENSValidator(contractAddr, newTestResolver(ensClient), signer.signContent)
 
 	// set up rpc and create resourcehandler with ENS sim backend
-	rh, _, _, teardownTest, err := setupTest(contractbackend, validator)
+	rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -430,27 +496,21 @@ func TestResourceENSOwner(t *testing.T) {
 	// create new resource when we are owner = ok
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	_, err = rh.NewResource(ctx, safeName, resourceFrequency)
+	rsrc, err := rh.NewResource(ctx, safeName, resourceFrequency)
 	if err != nil {
 		t.Fatalf("Create resource fail: %v", err)
 	}
 
 	data := []byte("foo")
-	// update resource when we are owner = ok
+	key := rh.resourceHash(1, 1, rsrc.nameHash)
-	_, err = rh.Update(ctx, safeName, data)
+	digest := rh.keyDataHash(key, data)
+	sig, err := rh.signer.Sign(digest)
 	if err != nil {
-		t.Fatalf("Update resource fail: %v", err)
+		t.Fatalf("sign fail: %v", err)
 	}
-
+	chunk := newUpdateChunk(key, &sig, 1, 1, safeName, data, len(data))
-	// update resource when we are owner = ok
+	if !rh.Validate(chunk.Key, chunk.SData) {
-	signertwo, err := newTestSigner()
+		t.Fatal("Chunk validator fail")
-	if err != nil {
-		t.Fatal(err)
-	}
-	rh.validator.(*ENSValidator).signFunc = signertwo.signContent
-	_, err = rh.Update(ctx, safeName, data)
-	if err == nil {
-		t.Fatalf("Expected resource update fail due to owner mismatch")
 	}
 }
 
@@ -462,7 +522,7 @@ func fwdBlocks(count int, backend *fakeBackend) {
 }
 
 // create rpc and resourcehandler
-func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(), err error) {
+func setupTest(backend headerGetter, ensBackend *ens.ENS, signer ResourceSigner) (rh *ResourceHandler, datadir string, teardown func(), err error) {
 
 	var fsClean func()
 	var rpcClean func()
@@ -478,14 +538,22 @@ func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceH
 	// temp datadir
 	datadir, err = ioutil.TempDir("", "rh")
 	if err != nil {
-		return nil, "", nil, nil, err
+		return nil, "", nil, err
 	}
 	fsClean = func() {
 		os.RemoveAll(datadir)
 	}
 
-	rh, err = NewTestResourceHandler(datadir, backend, validator, &ResourceLookupParams{Limit: false})
+	rhparams := &ResourceHandlerParams{
-	return rh, datadir, signer, cleanF, nil
+		QueryMaxPeriods: &ResourceLookupParams{
+			Limit: false,
+		},
+		Signer:    signer,
+		EthClient: backend,
+		EnsClient: ensBackend,
+	}
+	rh, err = NewTestResourceHandler(datadir, rhparams)
+	return rh, datadir, cleanF, nil
 }
 
 // Set up simulated ENS backend for use with ENSResourceHandler tests
@@ -531,55 +599,18 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string,
 	return contractAddress, contractBackend, nil
 }
 
-// implementation of an external signer to pass to validator
+func newTestSigner() (*GenericResourceSigner, error) {
-type testSigner struct {
-	privKey     *ecdsa.PrivateKey
-	hasher      SwarmHash
-	signContent SignFunc
-}
-
-func newTestSigner() (*testSigner, error) {
 	privKey, err := crypto.GenerateKey()
 	if err != nil {
 		return nil, err
 	}
-	return &testSigner{
+	return &GenericResourceSigner{
-		privKey:     privKey,
+		PrivKey: privKey,
-		hasher:      testHasher,
-		signContent: NewGenericResourceSigner(privKey),
 	}, nil
 }
 
-// Default fallthrough validation of mutable resource ownership
-type testValidator struct {
-	*baseValidator
-	hashFunc func(string) common.Hash
-}
-
-func newTestValidator(signFunc SignFunc) *testValidator {
-	return &testValidator{
-		baseValidator: &baseValidator{
-			signFunc: signFunc,
-		},
-		hashFunc: func(name string) common.Hash {
-			testHasher.Reset()
-			testHasher.Write([]byte(name))
-			return common.BytesToHash(testHasher.Sum(nil))
-		},
-	}
-
-}
-
-func (self *testValidator) checkAccess(name string, address common.Address) (bool, error) {
-	return true, nil
-}
-
-func (self *testValidator) NameHash(name string) common.Hash {
-	return self.hashFunc(name)
-}
-
 func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) {
-	chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(key)
+	chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(key)
 	if err != nil {
 		return nil, err
 	}
@@ -589,18 +620,3 @@ func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) {
 	}
 	return data, nil
 }
-
-type testResolver struct {
-	ens *ens.ENS
-}
-
-func newTestResolver(ens *ens.ENS) *testResolver {
-	return &testResolver{
-		ens: ens,
-	}
-}
-
-func (r *testResolver) ValidateOwner(name string, address common.Address) (bool, error) {
-	addr, err := r.ens.Owner(ens.EnsNode(name))
-	return addr == address, err
-}

swarm/storage/types.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/bmt"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto/sha3"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 const MaxPO = 7
@@ -180,18 +181,18 @@ type Chunk struct {
 	dbStoredC  chan bool // never remove a chunk from memStore before it is written to dbStore
 	dbStored   bool
 	dbStoredMu *sync.Mutex
-	errored    bool // flag which is set when the chunk request has errored or timeouted
+	errored    error // flag which is set when the chunk request has errored or timeouted
 	erroredMu  sync.Mutex
 }
 
-func (c *Chunk) SetErrored(val bool) {
+func (c *Chunk) SetErrored(err error) {
 	c.erroredMu.Lock()
 	defer c.erroredMu.Unlock()
 
-	c.errored = val
+	c.errored = err
 }
 
-func (c *Chunk) GetErrored() bool {
+func (c *Chunk) GetErrored() error {
 	c.erroredMu.Lock()
 	defer c.erroredMu.Unlock()
 
@@ -257,6 +258,31 @@ func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
 	return self.SectionReader.Size(), nil
 }
 
+type StoreParams struct {
+	Hash          SwarmHasher `toml:"-"`
+	DbCapacity    uint64
+	CacheCapacity uint
+	BaseKey       []byte
+}
+
+func NewStoreParams(capacity uint64, hash SwarmHasher, basekey []byte) *StoreParams {
+	if basekey == nil {
+		basekey = make([]byte, 32)
+	}
+	if hash == nil {
+		hash = MakeHashFunc("SHA3")
+	}
+	if capacity == 0 {
+		capacity = defaultDbCapacity
+	}
+	return &StoreParams{
+		Hash:          hash,
+		DbCapacity:    capacity,
+		CacheCapacity: defaultCacheCapacity,
+		BaseKey:       basekey,
+	}
+}
+
 type ChunkData []byte
 
 type Reference []byte
@@ -285,3 +311,34 @@ func (c ChunkData) Size() int64 {
 func (c ChunkData) Data() []byte {
 	return c[8:]
 }
+
+type ChunkValidator interface {
+	Validate(key Key, data []byte) bool
+}
+
+// Provides method for validation of content address in chunks
+// Holds the corresponding hasher to create the address
+type ContentAddressValidator struct {
+	Hasher SwarmHasher
+}
+
+// Constructor
+func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
+	return &ContentAddressValidator{
+		Hasher: hasher,
+	}
+}
+
+// Validate that the given key is a valid content address for the given data
+func (self *ContentAddressValidator) Validate(key Key, data []byte) bool {
+	hasher := self.Hasher()
+	hasher.ResetWithLength(data[:8])
+	hasher.Write(data[8:])
+	hash := hasher.Sum(nil)
+
+	if !bytes.Equal(hash, key[:]) {
+		log.Error("invalid content address", "expected", fmt.Sprintf("%x", hash), "have", key)
+		return false
+	}
+	return true
+}

swarm/swarm.go

@@ -117,15 +117,6 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
 	}
 	log.Debug(fmt.Sprintf("Setting up Swarm service components"))
 
-	hash := storage.MakeHashFunc(config.DPAParams.Hash)
-	self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.FromHex(config.BzzKey), mockStore)
-	if err != nil {
-		return
-	}
-
-	// setup local store
-	log.Debug(fmt.Sprintf("Set up local storage"))
-
 	kp := network.NewKadParams()
 	to := network.NewKademlia(
 		common.FromHex(config.BzzKey),
@@ -146,40 +137,16 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
 		HiveParams:   config.HiveParams,
 	}
 
-	db := storage.NewDBAPI(self.lstore)
-	delivery := stream.NewDelivery(to, db)
 	// TODO: decide on intervals store file location
 	stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db"))
 	if err != nil {
 		return
 	}
-	self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{
-		DoSync:          true,
-		DoRetrieve:      true,
-		SyncUpdateDelay: config.SyncUpdateDelay,
-	})
-
-	self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)
-
-	// set up DPA, the cloud storage local access layer
-	dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
-	log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
-	// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
-	self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams)
-	log.Debug(fmt.Sprintf("-> Content Store API"))
-
-	// Pss = postal service over swarm (devp2p over bzz)
-	if self.config.PssEnabled {
-		pssparams := pss.NewPssParams(self.privateKey)
-		self.ps = pss.NewPss(to, pssparams)
-		if pss.IsActiveHandshake {
-			pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
-		}
-	}
 
 	// set up high level api
 	//transactOpts := bind.NewKeyedTransactor(self.privateKey)
 	var resolver *api.MultiResolver
+	var ensresolver *ens.ENS
 	if len(config.EnsAPIs) > 0 {
 		opts := []api.MultiResolverOption{}
 		for _, c := range config.EnsAPIs {
@@ -188,6 +155,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
 			if err != nil {
 				return nil, err
 			}
+			ensresolver = r.ENS
 			opts = append(opts, api.MultiResolverOptionWithResolver(r, tld))
 
 		}
@@ -195,19 +163,67 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
 		self.dns = resolver
 	}
 
+	self.lstore, err = storage.NewLocalStore(config.LocalStoreParams, mockStore)
+	if err != nil {
+		return
+	}
+
 	var resourceHandler *storage.ResourceHandler
 	// if use resource updates
+
 	if self.config.ResourceEnabled && resolver != nil {
-		resourceparams := &storage.ResourceHandlerParams{
+		resolver.SetNameHash(ens.EnsNode)
-			Validator: storage.NewENSValidator(config.EnsRoot, resolver, storage.NewGenericResourceSigner(self.privateKey)),
+		rhparams := &storage.ResourceHandlerParams{
+			// TODO: config parameter to set limits
+			QueryMaxPeriods: &storage.ResourceLookupParams{
+				Limit: false,
+			},
+			Signer: &storage.GenericResourceSigner{
+				PrivKey: self.privateKey,
+			},
+			EthClient: resolver,
+			EnsClient: ensresolver,
 		}
-		resolver.SetNameHash(resourceparams.Validator.NameHash)
+		resourceHandler, err = storage.NewResourceHandler(rhparams)
-		hashfunc := storage.MakeHashFunc(storage.SHA3Hash)
-		chunkStore := storage.NewResourceChunkStore(self.lstore, func(*storage.Chunk) error { return nil })
-		resourceHandler, err = storage.NewResourceHandler(hashfunc, chunkStore, resolver, resourceparams)
 		if err != nil {
 			return nil, err
 		}
+		resourceHandler.SetStore(self.lstore)
+	}
+
+	var validators []storage.ChunkValidator
+	validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.SHA3Hash)))
+	if resourceHandler != nil {
+		validators = append(validators, resourceHandler)
+	}
+	self.lstore.Validators = validators
+
+	// setup local store
+	log.Debug(fmt.Sprintf("Set up local storage"))
+
+	db := storage.NewDBAPI(self.lstore)
+	delivery := stream.NewDelivery(to, db)
+
+	self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{
+		DoSync:          true,
+		DoRetrieve:      true,
+		SyncUpdateDelay: config.SyncUpdateDelay,
+	})
+
+	// set up DPA, the cloud storage local access layer
+	dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
+	log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
+	// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
+	self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams)
+	log.Debug(fmt.Sprintf("-> Content Store API"))
+
+	self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)
+
+	// Pss = postal service over swarm (devp2p over bzz)
+	pssparams := pss.NewPssParams(self.privateKey)
+	self.ps = pss.NewPss(to, pssparams)
+	if pss.IsActiveHandshake {
+		pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
 	}
 
 	self.api = api.NewApi(self.dpa, self.dns, resourceHandler)

swarm/testutil/http.go

@@ -47,12 +47,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
 	if err != nil {
 		t.Fatal(err)
 	}
-	storeparams := &storage.StoreParams{
+	storeparams := storage.NewDefaultLocalStoreParams()
-		ChunkDbPath:   dir,
+	storeparams.DbCapacity = 5000000
-		DbCapacity:    5000000,
+	storeparams.CacheCapacity = 5000
-		CacheCapacity: 5000,
+	storeparams.Init(dir)
-	}
+	localStore, err := storage.NewLocalStore(storeparams, nil)
-	localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, make([]byte, 32), nil)
 	if err != nil {
 		os.RemoveAll(dir)
 		t.Fatal(err)
@@ -64,8 +63,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	rhparams := &storage.ResourceHandlerParams{
-	rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil, &storage.ResourceLookupParams{Limit: false})
+		QueryMaxPeriods: &storage.ResourceLookupParams{},
+		EthClient:       &fakeBackend{},
+	}
+	rh, err := storage.NewTestResourceHandler(resourceDir, rhparams)
 	if err != nil {
 		t.Fatal(err)
 	}

vendor/github.com/naoina/toml/encode.go

@@ -61,20 +61,26 @@ func (cfg *Config) NewEncoder(w io.Writer) *Encoder {
 // Encode writes the TOML of v to the stream.
 // See the documentation for Marshal for details about the conversion of Go values to TOML.
 func (e *Encoder) Encode(v interface{}) error {
-	rv := reflect.ValueOf(v)
+	var (
+		buf = &tableBuf{typ: ast.TableTypeNormal}
+		rv  = reflect.ValueOf(v)
+		err error
+	)
+
 	for rv.Kind() == reflect.Ptr {
 		if rv.IsNil() {
 			return &marshalNilError{rv.Type()}
 		}
 		rv = rv.Elem()
 	}
-	buf := &tableBuf{typ: ast.TableTypeNormal}
+
-	var err error
 	switch rv.Kind() {
 	case reflect.Struct:
 		err = buf.structFields(e.cfg, rv)
 	case reflect.Map:
 		err = buf.mapFields(e.cfg, rv)
+	case reflect.Interface:
+		return e.Encode(rv.Interface())
 	default:
 		err = &marshalTableError{rv.Type()}
 	}

vendor/github.com/naoina/toml/parse.go

@@ -97,6 +97,7 @@ type toml struct {
 	currentTable *ast.Table
 	s            string
 	key          string
+	tableKeys    []string
 	val          ast.Value
 	arr          *array
 	stack        []*stack
@@ -180,12 +181,12 @@ func (p *tomlParser) SetArray(begin, end int) {
 }
 
 func (p *toml) SetTable(buf []rune, begin, end int) {
-	p.setTable(p.table, buf, begin, end)
+	rawName := string(buf[begin:end])
+	p.setTable(p.table, rawName, p.tableKeys)
+	p.tableKeys = nil
 }
 
-func (p *toml) setTable(parent *ast.Table, buf []rune, begin, end int) {
+func (p *toml) setTable(parent *ast.Table, name string, names []string) {
-	name := string(buf[begin:end])
-	names := splitTableKey(name)
 	parent, err := p.lookupTable(parent, names[:len(names)-1])
 	if err != nil {
 		p.Error(err)
@@ -230,12 +231,12 @@ func (p *tomlParser) SetTableString(begin, end int) {
 }
 
 func (p *toml) SetArrayTable(buf []rune, begin, end int) {
-	p.setArrayTable(p.table, buf, begin, end)
+	rawName := string(buf[begin:end])
+	p.setArrayTable(p.table, rawName, p.tableKeys)
+	p.tableKeys = nil
 }
 
-func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
+func (p *toml) setArrayTable(parent *ast.Table, name string, names []string) {
-	name := string(buf[begin:end])
-	names := splitTableKey(name)
 	parent, err := p.lookupTable(parent, names[:len(names)-1])
 	if err != nil {
 		p.Error(err)
@@ -260,11 +261,11 @@ func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
 func (p *toml) StartInlineTable() {
 	p.skip = false
 	p.stack = append(p.stack, &stack{p.key, p.currentTable})
-	buf := []rune(p.key)
+	names := []string{p.key}
 	if p.arr == nil {
-		p.setTable(p.currentTable, buf, 0, len(buf))
+		p.setTable(p.currentTable, names[0], names)
 	} else {
-		p.setArrayTable(p.currentTable, buf, 0, len(buf))
+		p.setArrayTable(p.currentTable, names[0], names)
 	}
 }
 
@@ -282,6 +283,13 @@ func (p *toml) AddLineCount(i int) {
 
 func (p *toml) SetKey(buf []rune, begin, end int) {
 	p.key = string(buf[begin:end])
+	if len(p.key) > 0 && p.key[0] == '"' {
+		p.key = p.unquote(p.key)
+	}
+}
+
+func (p *toml) AddTableKey() {
+	p.tableKeys = append(p.tableKeys, p.key)
 }
 
 func (p *toml) AddKeyValue() {
@@ -352,25 +360,3 @@ func (p *toml) lookupTable(t *ast.Table, keys []string) (*ast.Table, error) {
 	}
 	return t, nil
 }
-
-func splitTableKey(tk string) []string {
-	key := make([]byte, 0, 1)
-	keys := make([]string, 0, 1)
-	inQuote := false
-	for i := 0; i < len(tk); i++ {
-		k := tk[i]
-		switch {
-		case k == tableSeparator && !inQuote:
-			keys = append(keys, string(key))
-			key = key[:0] // reuse buffer.
-		case k == '"':
-			inQuote = !inQuote
-		case (k == ' ' || k == '\t') && !inQuote:
-			// skip.
-		default:
-			key = append(key, k)
-		}
-	}
-	keys = append(keys, string(key))
-	return keys
-}

vendor/github.com/naoina/toml/parse.peg

@@ -29,7 +29,7 @@ key <- bareKey / quotedKey
 
 bareKey <- <[0-9A-Za-z\-_]+> { p.SetKey(p.buffer, begin, end) }
 
-quotedKey <- '"' <basicChar+> '"' { p.SetKey(p.buffer, begin-1, end+1) }
+quotedKey <- < '"' basicChar* '"' > { p.SetKey(p.buffer, begin, end) }
 
 val <- (
     <datetime>    { p.SetTime(begin, end) }
@@ -55,7 +55,9 @@ inlineTable <- (
 
 inlineTableKeyValues <- (keyval inlineTableValSep?)*
 
-tableKey <- key (tableKeySep key)*
+tableKey <- tableKeyComp (tableKeySep tableKeyComp)*
+
+tableKeyComp <- key { p.AddTableKey() }
 
 tableKeySep <- ws '.' ws
 
@@ -117,7 +119,7 @@ timeNumoffset <- [\-+] timeHour ':' timeMinute
 timeOffset <- 'Z' / timeNumoffset
 partialTime <- timeHour ':' timeMinute ':' timeSecond timeSecfrac?
 fullDate <- dateFullYear '-' dateMonth '-' dateMDay
-fullTime <- partialTime timeOffset
+fullTime <- partialTime timeOffset?
 datetime <- (fullDate ('T' fullTime)?) / partialTime
 
 digit <- [0-9]

vendor/vendor.json

@@ -292,10 +292,10 @@
 			"revisionTime": "2017-07-13T18:40:44Z"
 		},
 		{
-			"checksumSHA1": "2gmvVTDCks8cPhpmyDlvm0sbrXE=",
+			"checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=",
 			"path": "github.com/naoina/toml",
-			"revision": "ac014c6b6502388d89a85552b7208b8da7cfe104",
+			"revision": "9fafd69674167c06933b1787ae235618431ce87f",
-			"revisionTime": "2017-04-10T21:57:17Z"
+			"revisionTime": "2017-09-18T21:04:37Z"
 		},
 		{
 			"checksumSHA1": "xZBlSMT5o/A+EDOro6KbfHZwSNc=",