forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-18 09:53:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

resolved merge conflicts

Browse source
This commit is contained in:
Anton Evangelatov 2018-04-16 12:46:28 +03:00
commit 7c6b751209
39 changed files with 1522 additions and 1255 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cmd/swarm/config.go
View file

@ -76,7 +76,6 @@ const (
SWARM_ENV_STORE_PATH = "SWARM_STORE_PATH"
SWARM_ENV_STORE_CAPACITY = "SWARM_STORE_CAPACITY"
SWARM_ENV_STORE_CACHE_CAPACITY = "SWARM_STORE_CACHE_CAPACITY"
SWARM_ENV_STORE_RADIUS = "SWARM_STORE_RADIUS"
GETH_ENV_DATADIR = "GETH_DATADIR"
)
@ -237,19 +236,15 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
}
if storePath := ctx.GlobalString(SwarmStorePath.Name); storePath != "" {
currentConfig.StoreParams.ChunkDbPath = storePath
currentConfig.LocalStoreParams.ChunkDbPath = storePath
}
if storeCapacity := ctx.GlobalUint64(SwarmStoreCapacity.Name); storeCapacity != 0 {
currentConfig.StoreParams.DbCapacity = storeCapacity
currentConfig.LocalStoreParams.DbCapacity = storeCapacity
}
if storeCacheCapacity := ctx.GlobalUint(SwarmStoreCacheCapacity.Name); storeCacheCapacity != 0 {
currentConfig.StoreParams.CacheCapacity = storeCacheCapacity
}
if storeRadius := ctx.GlobalInt(SwarmStoreRadius.Name); storeRadius != 0 {
currentConfig.StoreParams.Radius = storeRadius
currentConfig.LocalStoreParams.CacheCapacity = storeCacheCapacity
}
return currentConfig

14
cmd/swarm/config_test.go
View file

@ -156,7 +156,7 @@ func TestConfigFileOverrides(t *testing.T) {
defaultConf.PssEnabled = true
defaultConf.NetworkId = 54
defaultConf.Port = httpPort
defaultConf.StoreParams.DbCapacity = 9000000
defaultConf.DbCapacity = 9000000
defaultConf.HiveParams.KeepAliveInterval = 6000000000
defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
//defaultConf.SyncParams.KeyBufferSize = 512
@ -232,7 +232,7 @@ func TestConfigFileOverrides(t *testing.T) {
t.Fatal("Expected Pss to be enabled, but is false")
}
if info.StoreParams.DbCapacity != 9000000 {
if info.DbCapacity != 9000000 {
t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
}
@ -368,7 +368,7 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
defaultConf.PssEnabled = false
defaultConf.NetworkId = 54
defaultConf.Port = "8588"
defaultConf.StoreParams.DbCapacity = 9000000
defaultConf.DbCapacity = 9000000
defaultConf.HiveParams.KeepAliveInterval = 6000000000
defaultConf.Swap.Params.Strategy.AutoCashInterval = 600 * time.Second
//defaultConf.SyncParams.KeyBufferSize = 512
@ -378,10 +378,12 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
t.Fatalf("Error creating TOML file in TestFileOverride: %v", err)
}
//write file
f, err := ioutil.TempFile("", "testconfig.toml")
fname := "testconfig.toml"
f, err := ioutil.TempFile("", fname)
if err != nil {
t.Fatalf("Error writing TOML file in TestFileOverride: %v", err)
}
defer os.Remove(fname)
//write file
_, err = f.WriteString(string(out))
if err != nil {
@ -446,8 +448,8 @@ func TestConfigCmdLineOverridesFile(t *testing.T) {
t.Fatal("Expected Sync to be enabled, but is false")
}
if info.StoreParams.DbCapacity != 9000000 {
t.Fatalf("Expected network ID to be %d, got %d", 54, info.NetworkId)
if info.LocalStoreParams.DbCapacity != 9000000 {
t.Fatalf("Expected Capacity to be %d, got %d", 9000000, info.LocalStoreParams.DbCapacity)
}
if info.HiveParams.KeepAliveInterval != 6000000000 {

6
cmd/swarm/db.go
View file

@ -112,6 +112,8 @@ func openLDBStore(path string, basekey []byte) (*storage.LDBStore, error) {
if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
return nil, fmt.Errorf("invalid chunkdb path: %s", err)
}
hash := storage.MakeHashFunc("SHA3")
return storage.NewLDBStore(path, hash, 10000000, func(k storage.Key) (ret uint8) { return uint8(storage.Proximity(basekey[:], k[:])) })
storeparams := storage.NewDefaultStoreParams()
ldbparams := storage.NewLDBStoreParams(storeparams, path)
return storage.NewLDBStore(ldbparams)
}

6
cmd/swarm/main.go
View file

@ -169,11 +169,6 @@ var (
Usage: "Number of recent chunks cached in memory (default 5000)",
EnvVar: SWARM_ENV_STORE_CACHE_CAPACITY,
}
SwarmStoreRadius = cli.IntFlag{
Name: "store.radius",
Usage: "Minimum proximity order (number of identical prefix bits of address key) for chunks to warrant storage (default 0)",
EnvVar: SWARM_ENV_STORE_RADIUS,
}
// the following flags are deprecated and should be removed in the future
DeprecatedEthAPIFlag = cli.StringFlag{
@ -385,7 +380,6 @@ Remove corrupt entries from a local chunk database.
SwarmStorePath,
SwarmStoreCapacity,
SwarmStoreCacheCapacity,
SwarmStoreRadius,
//deprecated flags
DeprecatedEthAPIFlag,
DeprecatedEnsAddrFlag,

2
swarm/api/api.go
View file

@ -617,7 +617,7 @@ func (self *Api) ResourceUpdate(ctx context.Context, name string, data []byte) (
}
func (self *Api) ResourceHashSize() int {
return self.resource.HashSize()
return self.resource.HashSize
}
func (self *Api) ResourceIsValidated() bool {

7
swarm/api/config.go
View file

@ -42,8 +42,8 @@ const (
// allow several bzz nodes running in parallel
type Config struct {
// serialised/persisted fields
*storage.StoreParams
*storage.DPAParams
*storage.LocalStoreParams
*network.HiveParams
Swap *swap.SwapParams
//*network.SyncParams
@ -72,7 +72,7 @@ type Config struct {
func NewConfig() (self *Config) {
self = &Config{
StoreParams: storage.NewDefaultStoreParams(),
LocalStoreParams: storage.NewDefaultLocalStoreParams(),
DPAParams: storage.NewDPAParams(),
HiveParams: network.NewHiveParams(),
//SyncParams: network.NewDefaultSyncParams(),
@ -117,8 +117,9 @@ func (self *Config) Init(prvKey *ecdsa.PrivateKey) {
if self.SwapEnabled {
self.Swap.Init(self.Contract, prvKey)
}
self.privateKey = prvKey
self.StoreParams.Init(self.Path)
self.LocalStoreParams.Init(self.Path)
}
func (self *Config) ShiftPrivateKey() (privKey *ecdsa.PrivateKey) {

3
swarm/api/config_test.go
View file

@ -36,6 +36,7 @@ func TestConfig(t *testing.T) {
one := NewConfig()
two := NewConfig()
one.LocalStoreParams = two.LocalStoreParams
if equal := reflect.DeepEqual(one, two); !equal {
t.Fatal("Two default configs are not equal")
}
@ -52,7 +53,7 @@ func TestConfig(t *testing.T) {
if one.Swap.PayProfile.Beneficiary == (common.Address{}) && one.SwapEnabled {
t.Fatal("Failed to correctly initialize SwapParams")
}
if one.StoreParams.ChunkDbPath == one.Path {
if one.ChunkDbPath == one.Path {
t.Fatal("Failed to correctly initialize StoreParams")
}
}

4
swarm/api/http/server.go
View file

@ -481,13 +481,13 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
code := 0
defaultErr := fmt.Errorf("%s: %v", supErr, err)
rsrcErr, ok := err.(*storage.ResourceError)
if !ok {
if !ok && rsrcErr != nil {
code = rsrcErr.Code()
}
switch code {
case storage.ErrInvalidValue:
return http.StatusBadRequest, defaultErr
case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn:
case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn, storage.ErrInit:
return http.StatusNotFound, defaultErr
case storage.ErrUnauthorized, storage.ErrInvalidSignature:
return http.StatusUnauthorized, defaultErr

37
swarm/api/http/server_test.go
View file

@ -29,6 +29,7 @@ import (
"strings"
"testing"
"github.com/ethereum/go-ethereum/contracts/ens"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/api"
swarm "github.com/ethereum/go-ethereum/swarm/api/client"
@ -37,11 +38,9 @@ import (
)
func init() {
verbose := flag.Bool("v", false, "verbose")
loglevel := flag.Int("loglevel", 2, "loglevel")
flag.Parse()
if *verbose {
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
type resourceResponse struct {
@ -55,10 +54,8 @@ func TestBzzResource(t *testing.T) {
defer srv.Close()
// our mutable resource "name"
keybytes := []byte("foo")
srv.Hasher.Reset()
srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes)))
keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil))
keybytes := "foo"
keybyteshash := ens.EnsNode(keybytes)
// data of update 1
databytes := make([]byte, 666)
@ -68,7 +65,7 @@ func TestBzzResource(t *testing.T) {
}
// creates resource and sets update 1
url := fmt.Sprintf("%s/bzz-resource:/%x/13", srv.URL, keybytes)
url := fmt.Sprintf("%s/bzz-resource:/%s/13", srv.URL, []byte(keybytes))
resp, err := http.Post(url, "application/octet-stream", bytes.NewReader(databytes))
if err != nil {
t.Fatal(err)
@ -86,8 +83,8 @@ func TestBzzResource(t *testing.T) {
if err != nil {
t.Fatalf("data %s could not be unmarshaled: %v", b, err)
}
if rsrcResp.Update.Hex() != keybyteshash {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash, rsrcResp.Resource)
if !bytes.Equal(rsrcResp.Update, keybyteshash.Bytes()) {
t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash.Hex(), rsrcResp.Update.Hex())
}
// get manifest
@ -131,8 +128,16 @@ func TestBzzResource(t *testing.T) {
t.Fatal(err)
}
// get non-existent name
url = fmt.Sprintf("%s/bzz-resource:/bar", srv.URL)
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
}
resp.Body.Close()
// get latest update (1.1) through resource directly
url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
@ -150,7 +155,7 @@ func TestBzzResource(t *testing.T) {
}
// update 2
url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
data := []byte("foo")
resp, err = http.Post(url, "application/octet-stream", bytes.NewReader(data))
if err != nil {
@ -162,7 +167,7 @@ func TestBzzResource(t *testing.T) {
}
// get latest update (1.2) through resource directly
url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes)
url = fmt.Sprintf("%s/bzz-resource:/%s", srv.URL, keybytes)
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
@ -180,7 +185,7 @@ func TestBzzResource(t *testing.T) {
}
// get latest update (1.2) with specified period
url = fmt.Sprintf("%s/bzz-resource:/%x/1", srv.URL, keybytes)
url = fmt.Sprintf("%s/bzz-resource:/%s/1", srv.URL, keybytes)
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)
@ -198,7 +203,7 @@ func TestBzzResource(t *testing.T) {
}
// get first update (1.1) with specified period and version
url = fmt.Sprintf("%s/bzz-resource:/%x/1/1", srv.URL, keybytes)
url = fmt.Sprintf("%s/bzz-resource:/%s/1/1", srv.URL, keybytes)
resp, err = http.Get(url)
if err != nil {
t.Fatal(err)

16
swarm/fuse/swarmfs_test.go
View file

@ -21,6 +21,7 @@ package fuse
import (
"bytes"
"crypto/rand"
"flag"
"io"
"io/ioutil"
"os"
@ -29,8 +30,23 @@ import (
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/log"
colorable "github.com/mattn/go-colorable"
)
var (
loglevel = flag.Int("loglevel", 4, "verbosity of logs")
rawlog = flag.Bool("rawlog", false, "turn off terminal formatting in logs")
)
func init() {
flag.Parse()
log.PrintOrigins(true)
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(!*rawlog))))
}
type fileInfo struct {
perm uint64
uid int

6
swarm/network/stream/common_test.go
View file

@ -133,7 +133,11 @@ func newStreamerTester(t *testing.T) (*p2ptest.ProtocolTester, *Registry, *stora
os.RemoveAll(datadir)
}
localStore, err := storage.NewTestLocalStoreForAddr(datadir, addr.Over())
params := storage.NewDefaultLocalStoreParams()
params.Init(datadir)
params.BaseKey = addr.Over()
localStore, err := storage.NewTestLocalStoreForAddr(params)
if err != nil {
return nil, nil, nil, removeDataDir, err
}

12
swarm/network/stream/delivery.go
View file

@ -138,7 +138,7 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e
if created {
if err := d.RequestFromPeers(chunk.Key[:], false, sp.ID()); err != nil {
log.Warn("unable to forward chunk request", "peer", sp.ID(), "key", chunk.Key, "err", err)
chunk.SetErrored(true)
chunk.SetErrored(storage.ErrChunkForward)
return nil
}
}
@ -149,10 +149,10 @@ func (d *Delivery) handleRetrieveRequestMsg(sp *Peer, req *RetrieveRequestMsg) e
select {
case <-chunk.ReqC:
case <-t.C:
chunk.SetErrored(true)
chunk.SetErrored(storage.ErrChunkTimeout)
return
}
chunk.SetErrored(false)
chunk.SetErrored(nil)
if req.SkipCheck {
err := sp.Deliver(chunk, s.priority)
@ -209,6 +209,12 @@ R:
chunk.SData = req.SData
d.db.Put(chunk)
chunk.WaitToStore()
err = chunk.GetErrored()
if err != nil {
if err == storage.ErrChunkInvalid {
req.peer.Drop(err)
}
}
close(chunk.ReqC)
}
}

5
swarm/network/stream/snapshot_sync_test.go
View file

@ -662,7 +662,10 @@ func createTestLocalStorageForId(id discover.NodeID, addr *network.BzzAddr) (sto
}
datadirs[id] = datadir
var store storage.ChunkStore
store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
params := storage.NewDefaultLocalStoreParams()
params.ChunkDbPath = datadir
params.BaseKey = addr.Over()
store, err = storage.NewTestLocalStoreForAddr(params)
if err != nil {
return nil, err
}

5
swarm/network/stream/testing/testing.go
View file

@ -55,7 +55,10 @@ func SetStores(addrs ...network.Addr) ([]storage.ChunkStore, func(), error) {
break
}
var store storage.ChunkStore
store, err = storage.NewTestLocalStoreForAddr(datadir, addr.Over())
params := storage.NewDefaultLocalStoreParams()
params.Init(datadir)
params.BaseKey = addr.Over()
store, err = storage.NewTestLocalStoreForAddr(params)
if err != nil {
break
}

17
swarm/storage/dpa.go
View file

@ -17,7 +17,6 @@
package storage
import (
"errors"
"io"
"time"
)
@ -41,8 +40,6 @@ const (
)
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrFetching = errors.New("chunk still fetching")
// timeout interval before retrieval is timed out
searchTimeout = 3 * time.Second
)
@ -64,18 +61,14 @@ func NewDPAParams() *DPAParams {
// for testing locally
func NewLocalDPA(datadir string, basekey []byte) (*DPA, error) {
hash := MakeHashFunc("SHA3")
dbStore, err := NewLDBStore(datadir, hash, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
params := NewDefaultLocalStoreParams()
params.Init(datadir)
localStore, err := NewLocalStore(params, nil)
if err != nil {
return nil, err
}
return NewDPA(&LocalStore{
memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity),
DbStore: dbStore,
}, NewDPAParams()), nil
localStore.Validators = append(localStore.Validators, NewContentAddressValidator(MakeHashFunc(SHA3Hash)))
return NewDPA(localStore, NewDPAParams()), nil
}
func NewDPA(store ChunkStore, params *DPAParams) *DPA {

10
swarm/storage/dpa_test.go
View file

@ -32,14 +32,14 @@ func TestDPArandom(t *testing.T) {
}
func testDpaRandom(toEncrypt bool, t *testing.T) {
tdb, err := newTestDbStore(false)
tdb, err := newTestDbStore(false, false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer tdb.close()
db := tdb.LDBStore
db.setCapacity(50000)
memStore := NewMemStore(db, defaultCacheCapacity, defaultChunkRequestsCacheCapacity)
memStore := NewMemStore(NewDefaultStoreParams(), db)
localStore := &LocalStore{
memStore: memStore,
DbStore: db,
@ -71,7 +71,7 @@ func testDpaRandom(toEncrypt bool, t *testing.T) {
}
ioutil.WriteFile("/tmp/slice.bzz.16M", slice, 0666)
ioutil.WriteFile("/tmp/result.bzz.16M", resultSlice, 0666)
localStore.memStore = NewMemStore(db, defaultCacheCapacity, defaultChunkRequestsCacheCapacity)
localStore.memStore = NewMemStore(NewDefaultStoreParams(), db)
resultReader, isEncrypted = dpa.Retrieve(key)
if isEncrypted != toEncrypt {
t.Fatalf("isEncrypted expected %v got %v", toEncrypt, isEncrypted)
@ -97,13 +97,13 @@ func TestDPA_capacity(t *testing.T) {
}
func testDPA_capacity(toEncrypt bool, t *testing.T) {
tdb, err := newTestDbStore(false)
tdb, err := newTestDbStore(false, false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer tdb.close()
db := tdb.LDBStore
memStore := NewMemStore(db, 0, 0)
memStore := NewMemStore(NewDefaultStoreParams(), db)
localStore := &LocalStore{
memStore: memStore,
DbStore: db,

16
swarm/storage/error.go
View file

@ -1,7 +1,12 @@
package storage
import (
"errors"
)
const (
ErrNotFound = iota
ErrInit = iota
ErrNotFound
ErrIO
ErrUnauthorized
ErrInvalidValue
@ -12,3 +17,12 @@ const (
ErrPeriodDepth
ErrCnt
)
var (
ErrChunkNotFound = errors.New("chunk not found")
ErrFetching = errors.New("chunk still fetching")
ErrChunkInvalid = errors.New("invalid chunk")
ErrChunkForward = errors.New("cannot forward")
ErrChunkUnavailable = errors.New("chunk unavailable")
ErrChunkTimeout = errors.New("timeout")
)

50
swarm/storage/ldbstore.go
View file

@ -70,6 +70,21 @@ type gcItem struct {
po uint8
}
type LDBStoreParams struct {
*StoreParams
Path string
Po func(Key) uint8
}
// NewLDBStoreParams constructs LDBStoreParams with the specified values.
func NewLDBStoreParams(storeparams *StoreParams, path string) *LDBStoreParams {
return &LDBStoreParams{
StoreParams: storeparams,
Path: path,
Po: func(k Key) (ret uint8) { return uint8(Proximity(storeparams.BaseKey[:], k[:])) },
}
}
type LDBStore struct {
db *LDBDatabase
@ -87,7 +102,6 @@ type LDBStore struct {
batchesC chan struct{}
batch *leveldb.Batch
lock sync.RWMutex
trusted bool // if hash integity check is to be performed (for testing only)
// Functions encodeDataFunc is used to bypass
// the default functionality of DbStore with
@ -102,9 +116,9 @@ type LDBStore struct {
// TODO: Instead of passing the distance function, just pass the address from which distances are calculated
// to avoid the appearance of a pluggable distance metric and opportunities of bugs associated with providing
// a function different from the one that is actually used.
func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8) (s *LDBStore, err error) {
func NewLDBStore(params *LDBStoreParams) (s *LDBStore, err error) {
s = new(LDBStore)
s.hashfunc = hash
s.hashfunc = params.Hash
s.batchC = make(chan bool)
s.batchesC = make(chan struct{}, 1)
@ -113,13 +127,13 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui
// associate encodeData with default functionality
s.encodeDataFunc = encodeData
s.db, err = NewLDBDatabase(path)
s.db, err = NewLDBDatabase(params.Path)
if err != nil {
return nil, err
}
s.po = po
s.setCapacity(capacity)
s.po = params.Po
s.setCapacity(params.DbCapacity)
s.bucketCnt = make([]uint64, 0x100)
for i := 0; i < 0x100; i++ {
@ -143,15 +157,11 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui
return s, nil
}
func (self *LDBStore) SetTrusted() {
self.trusted = true
}
// NewMockDbStore creates a new instance of DbStore with
// mockStore set to a provided value. If mockStore argument is nil,
// this function behaves exactly as NewDbStore.
func NewMockDbStore(path string, hash SwarmHasher, capacity uint64, po func(Key) uint8, mockStore *mock.NodeStore) (s *LDBStore, err error) {
s, err = NewLDBStore(path, hash, capacity, po)
func NewMockDbStore(params *LDBStoreParams, mockStore *mock.NodeStore) (s *LDBStore, err error) {
s, err = NewLDBStore(params)
if err != nil {
return nil, err
}
@ -344,7 +354,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
continue
}
key, err := hex.DecodeString(hdr.Name)
keybytes, err := hex.DecodeString(hdr.Name)
if err != nil {
log.Warn("ignoring invalid chunk file", "name", hdr.Name, "err", err)
continue
@ -354,6 +364,7 @@ func (s *LDBStore) Import(in io.Reader) (int64, error) {
if err != nil {
return count, err
}
key := Key(keybytes)
chunk := NewChunk(key, nil)
chunk.SData = data[32:]
s.Put(chunk)
@ -631,19 +642,6 @@ func (s *LDBStore) get(key Key) (chunk *Chunk, err error) {
}
}
if !s.trusted {
data_mod := data[32:]
hasher := s.hashfunc()
hasher.Write(data_mod)
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key) {
log.Error("apparent key/hash mismatch", "hash", hash, "key", key[:])
s.delete(indx.Idx, getIndexKey(key), s.po(key))
log.Error("invalid chunk in database.")
}
}
chunk = NewChunk(key, nil)
chunk.markAsStored()
decodeData(data, chunk)

25
swarm/storage/ldbstore_test.go
View file

@ -37,21 +37,25 @@ type testDbStore struct {
dir string
}
func newTestDbStore(mock bool) (*testDbStore, error) {
func newTestDbStore(mock bool, trusted bool) (*testDbStore, error) {
dir, err := ioutil.TempDir("", "bzz-storage-test")
if err != nil {
return nil, err
}
var db *LDBStore
storeparams := NewDefaultStoreParams()
params := NewLDBStoreParams(storeparams, dir)
params.Po = testPoFunc
if mock {
globalStore := mem.NewGlobalStore()
addr := common.HexToAddress("0x5aaeb6053f3e94c9b9a09f33669435e7ef1beaed")
mockStore := globalStore.NewNodeStore(addr)
db, err = NewMockDbStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc, mockStore)
db, err = NewMockDbStore(params, mockStore)
} else {
db, err = NewLDBStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc)
db, err = NewLDBStore(params)
}
return &testDbStore{db, dir}, err
@ -71,17 +75,16 @@ func (db *testDbStore) close() {
}
func testDbStoreRandom(n int, processors int, chunksize int, mock bool, t *testing.T) {
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, true)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
testStoreRandom(db, processors, n, chunksize, t)
}
func testDbStoreCorrect(n int, processors int, chunksize int, mock bool, t *testing.T) {
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
@ -138,7 +141,7 @@ func TestMockDbStoreCorrect_8_5k(t *testing.T) {
}
func testDbStoreNotFound(t *testing.T, mock bool) {
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
@ -169,7 +172,7 @@ func testIterator(t *testing.T, mock bool) {
chunks = append(chunks, NewChunk(nil, nil))
}
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, false)
if err != nil {
t.Fatalf("init dbStore failed: %v", err)
}
@ -224,22 +227,20 @@ func TestMockIterator(t *testing.T) {
}
func benchmarkDbStorePut(n int, processors int, chunksize int, mock bool, b *testing.B) {
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, true)
if err != nil {
b.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
benchmarkStorePut(db, processors, n, chunksize, b)
}
func benchmarkDbStoreGet(n int, processors int, chunksize int, mock bool, b *testing.B) {
db, err := newTestDbStore(mock)
db, err := newTestDbStore(mock, true)
if err != nil {
b.Fatalf("init dbStore failed: %v", err)
}
defer db.close()
db.trusted = true
benchmarkStoreGet(db, processors, n, chunksize, b)
}

59
swarm/storage/localstore.go
View file

@ -19,6 +19,7 @@ package storage
import (
"encoding/binary"
"fmt"
"path/filepath"
"sync"
"github.com/ethereum/go-ethereum/log"
@ -30,57 +31,75 @@ var (
dbStorePutCounter = metrics.NewRegisteredCounter("storage.db.dbstore.put.count", nil)
)
type StoreParams struct {
type LocalStoreParams struct {
*StoreParams
ChunkDbPath string
DbCapacity uint64
CacheCapacity uint
Radius int
Validators []ChunkValidator `toml:"-"`
}
//create params with default values
func NewDefaultStoreParams() (self *StoreParams) {
return &StoreParams{
DbCapacity: defaultLDBCapacity,
CacheCapacity: defaultCacheCapacity,
func NewDefaultLocalStoreParams() *LocalStoreParams {
return &LocalStoreParams{
StoreParams: NewDefaultStoreParams(),
}
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *LocalStoreParams) Init(path string) {
if self.ChunkDbPath == "" {
self.ChunkDbPath = filepath.Join(path, "chunks")
}
}
// LocalStore is a combination of inmemory db over a disk persisted db
// implements a Get/Put with fallback (caching) logic using any 2 ChunkStores
type LocalStore struct {
Validators []ChunkValidator
memStore *MemStore
DbStore *LDBStore
mu sync.Mutex
}
// This constructor uses MemStore and DbStore as components
func NewLocalStore(hash SwarmHasher, params *StoreParams, basekey []byte, mockStore *mock.NodeStore) (*LocalStore, error) {
dbStore, err := NewMockDbStore(params.ChunkDbPath, hash, params.DbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }, mockStore)
func NewLocalStore(params *LocalStoreParams, mockStore *mock.NodeStore) (*LocalStore, error) {
ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath)
dbStore, err := NewMockDbStore(ldbparams, mockStore)
if err != nil {
return nil, err
}
return &LocalStore{
memStore: NewMemStore(dbStore, params.CacheCapacity, defaultChunkRequestsCacheCapacity),
memStore: NewMemStore(params.StoreParams, dbStore),
DbStore: dbStore,
Validators: params.Validators,
}, nil
}
func NewTestLocalStoreForAddr(path string, basekey []byte) (*LocalStore, error) {
hasher := MakeHashFunc("SHA3")
dbStore, err := NewLDBStore(path, hasher, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
func NewTestLocalStoreForAddr(params *LocalStoreParams) (*LocalStore, error) {
ldbparams := NewLDBStoreParams(params.StoreParams, params.ChunkDbPath)
dbStore, err := NewLDBStore(ldbparams)
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity),
memStore: NewMemStore(params.StoreParams, dbStore),
DbStore: dbStore,
Validators: params.Validators,
}
return localStore, nil
}
// LocalStore is itself a chunk store
// unsafe, in that the data is not integrity checked
func (self *LocalStore) Put(chunk *Chunk) {
valid := true
for _, v := range self.Validators {
if valid = v.Validate(chunk.Key, chunk.SData); valid {
break
}
}
if !valid {
chunk.SetErrored(ErrChunkInvalid)
chunk.dbStoredC <- false
return
}
log.Trace("localstore.put", "key", chunk.Key)
self.mu.Lock()
defer self.mu.Unlock()
@ -139,11 +158,11 @@ func (self *LocalStore) GetOrCreateRequest(key Key) (chunk *Chunk, created bool)
var err error
chunk, err = self.get(key)
if err == nil && !chunk.GetErrored() {
if err == nil && chunk.GetErrored() == nil {
log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v found locally", key))
return chunk, false
}
if err == ErrFetching && !chunk.GetErrored() {
if err == ErrFetching && chunk.GetErrored() == nil {
log.Trace(fmt.Sprintf("LocalStore.GetOrRetrieve: %v hit on an existing request %v", key, chunk.ReqC))
return chunk, false
}

153
swarm/storage/localstore_test.go Normal file
View file

@ -0,0 +1,153 @@
package storage
import (
"io/ioutil"
"os"
"sync"
"testing"
"github.com/ethereum/go-ethereum/contracts/ens"
)
var (
hashfunc = MakeHashFunc("SHA3")
)
// convenience generator for unique chunks
type randomChunkGenerator struct {
data []byte
hasher SwarmHash
}
func newRandomChunkGenerator(stem []byte) *randomChunkGenerator {
gen := &randomChunkGenerator{
data: make([]byte, 8),
hasher: hashfunc(),
}
gen.data = append(gen.data, stem...)
return gen
}
func (self *randomChunkGenerator) newChunk() *Chunk {
self.hasher.Reset()
self.hasher.Write(self.data)
chunk := NewChunk(self.hasher.Sum(nil), nil)
chunk.SData = make([]byte, len(self.data))
copy(chunk.SData, self.data)
self.data[0]++
return chunk
}
// put to localstore and wait for stored channel
// does not check delivery error state
func putChunks(store *LocalStore, chunks ...*Chunk) {
wg := sync.WaitGroup{}
wg.Add(len(chunks))
go func() {
for _, c := range chunks {
<-c.dbStoredC
wg.Done()
}
}()
for _, c := range chunks {
go store.Put(c)
}
wg.Wait()
}
// tests that the content address validator correctly checks the data
// tests that resource update chunks are passed through content address validator
// the test checking the resouce update validator internal correctness is found in resource_test.go
func TestValidator(t *testing.T) {
bogusData := []byte("00000000bar")
// set up localstore
datadir, err := ioutil.TempDir("", "storage-testvalidator")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(datadir)
params := NewDefaultLocalStoreParams()
params.Init(datadir)
store, err := NewLocalStore(params, nil)
if err != nil {
t.Fatal(err)
}
// check puts with no validators, both succeed
gen := newRandomChunkGenerator([]byte("foo"))
goodChunk := gen.newChunk()
badChunk := gen.newChunk()
badChunk.SData = bogusData
putChunks(store, goodChunk, badChunk)
if err := goodChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on good content address chunk in spite of no validation, but got: %s", err)
}
if err := badChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on bad content address chunk in spite of no validation, but got: %s", err)
}
// add content address validator and check puts
// bad should fail, good should pass
store.Validators = append(store.Validators, NewContentAddressValidator(hashfunc))
goodChunk = gen.newChunk()
badChunk = gen.newChunk()
badChunk.SData = bogusData
putChunks(store, goodChunk, badChunk)
if err := goodChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on good content address chunk with content address validator only, but got: %s", err)
}
if err := badChunk.GetErrored(); err == nil {
t.Fatal("expected error on bad content address chunk with content address validator only, but got nil")
}
// append resource validator to validators and check puts
// bad should fail, good should pass, resource should pass
rhParams := &ResourceHandlerParams{}
rh, err := NewResourceHandler(rhParams)
if err != nil {
t.Fatal(err)
}
store.Validators = append(store.Validators, rh)
goodChunk = gen.newChunk()
key := rh.resourceHash(42, 1, ens.EnsNode("xyzzy.eth"))
data := []byte("bar")
uglyChunk := newUpdateChunk(key, nil, 42, 1, "xyzzy.eth", data, len(data))
putChunks(store, goodChunk, badChunk, uglyChunk)
if err := goodChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on good content address chunk with both validators, but got: %s", err)
}
if err := badChunk.GetErrored(); err == nil {
t.Fatal("expected error on bad chunk address with both validators, but got nil")
}
if err := uglyChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on resource update chunk with both validators, but got: %s", err)
}
// (redundant check)
// use only resource validator, and check puts
// bad should fail, good should fail, resource should pass
store.Validators[0] = store.Validators[1]
store.Validators = store.Validators[:1]
goodChunk = gen.newChunk()
key = rh.resourceHash(42, 2, ens.EnsNode("xyzzy.eth"))
data = []byte("baz")
uglyChunk = newUpdateChunk(key, nil, 42, 2, "xyzzy.eth", data, len(data))
putChunks(store, goodChunk, badChunk, uglyChunk)
if goodChunk.GetErrored() == nil {
t.Fatal("expected error on good content address chunk with resource validator only, but got nil")
}
if badChunk.GetErrored() == nil {
t.Fatal("expected error on bad content address chunk with resource validator only, but got nil")
}
if err := uglyChunk.GetErrored(); err != nil {
t.Fatalf("expected no error on resource update chunk with resource validator only, but got: %s", err)
}
}

30
swarm/storage/memstore_lrucache.go
View file

@ -33,8 +33,8 @@ type MemStore struct {
disabled bool
}
func NewMemStore(_ *LDBStore, cacheCapacity uint, requestsCapacity uint) (m *MemStore) {
if cacheCapacity == 0 {
func NewMemStore(params *StoreParams, _ *LDBStore) (m *MemStore) {
if params.CacheCapacity == 0 {
return &MemStore{
disabled: true,
}
@ -44,12 +44,12 @@ func NewMemStore(_ *LDBStore, cacheCapacity uint, requestsCapacity uint) (m *Mem
v := value.(*Chunk)
<-v.dbStoredC
}
c, err := lru.NewWithEvict(int(cacheCapacity), onEvicted)
c, err := lru.NewWithEvict(int(params.CacheCapacity), onEvicted)
if err != nil {
panic(err)
}
r, err := lru.New(int(requestsCapacity))
r, err := lru.New(int(params.ChunkRequestsCacheCapacity))
if err != nil {
panic(err)
}
@ -98,8 +98,7 @@ func (m *MemStore) Put(c *Chunk) {
if c.ReqC != nil {
select {
case <-c.ReqC:
ok := c.GetErrored()
if !ok {
if c.GetErrored() != nil {
m.requests.Remove(string(c.Key))
return
}
@ -120,7 +119,24 @@ func (m *MemStore) setCapacity(n int) {
if n <= 0 {
m.disabled = true
} else {
m = NewMemStore(nil, uint(n), defaultChunkRequestsCacheCapacity)
onEvicted := func(key interface{}, value interface{}) {
v := value.(*Chunk)
<-v.dbStoredC
}
c, err := lru.NewWithEvict(n, onEvicted)
if err != nil {
panic(err)
}
r, err := lru.New(defaultChunkRequestsCacheCapacity)
if err != nil {
panic(err)
}
m = &MemStore{
cache: c,
requests: r,
}
}
}

5
swarm/storage/memstore_lrucache_test.go
View file

@ -34,7 +34,8 @@ func newLDBStore(t *testing.T) (*LDBStore, func()) {
}
log.Trace("memstore.tempdir", "dir", dir)
db, err := NewLDBStore(dir, MakeHashFunc(SHA3Hash), defaultLDBCapacity, testPoFunc)
ldbparams := NewLDBStoreParams(NewDefaultStoreParams(), dir)
db, err := NewLDBStore(ldbparams)
if err != nil {
t.Fatal(err)
}
@ -57,7 +58,7 @@ func TestMemStoreAndLDBStore(t *testing.T) {
cacheCap := 200
requestsCap := 200
memStore := NewMemStore(ldb, uint(cacheCap), uint(requestsCap))
memStore := NewMemStore(NewStoreParams(4000, 200, 200, nil, nil), nil)
tests := []struct {
n int // number of chunks to push to memStore

3
swarm/storage/memstore_test.go
View file

@ -19,7 +19,8 @@ package storage
import "testing"
func newTestMemStore() *MemStore {
return NewMemStore(nil, defaultCacheCapacity, defaultChunkRequestsCacheCapacity)
storeparams := NewDefaultStoreParams()
return NewMemStore(storeparams, nil)
}
func testMemStoreRandom(n int, processors int, chunksize int, t *testing.T) {

4
swarm/storage/memstore_tree.god
View file

@ -62,7 +62,9 @@ a hash prefix subtree containing subtrees or one storage entry (but never both)
(access[] is a binary tree inside the multi-bit leveled hash tree)
*/
func NewMemStore(d *LDBStore, capacity uint) (m *MemStore) {
func NewMemStore(params *StoreParams, d *LDBStore) (m *MemStore) {
capacity := params.CacheCapacity
m = &MemStore{}
m.memtree = newMemTree(memTreeFLW, nil, 0)
m.ldbStore = d

15
swarm/storage/netstore.go
View file

@ -17,7 +17,6 @@
package storage
import (
"path/filepath"
"time"
)
@ -57,7 +56,7 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
err := self.retrieve(chunk)
if err != nil {
// mark chunk request as failed so that we can retry it later
chunk.SetErrored(true)
chunk.SetErrored(ErrChunkUnavailable)
return nil, err
}
}
@ -69,22 +68,14 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) {
select {
case <-t.C:
// mark chunk request as failed so that we can retry
chunk.SetErrored(true)
chunk.SetErrored(ErrChunkNotFound)
return nil, ErrChunkNotFound
case <-chunk.ReqC:
}
chunk.SetErrored(false)
chunk.SetErrored(nil)
return chunk, nil
}
//this can only finally be set after all config options (file, cmd line, env vars)
//have been evaluated
func (self *StoreParams) Init(path string) {
if self.ChunkDbPath == "" {
self.ChunkDbPath = filepath.Join(path, "chunks")
}
}
// Put is the entrypoint for local store requests coming from storeLoop
func (self *NetStore) Put(chunk *Chunk) {
self.localStore.Put(chunk)

6
swarm/storage/netstore_test.go
View file

@ -80,8 +80,10 @@ func TestNetstoreFailedRequest(t *testing.T) {
if err != nil {
t.Fatal(err)
}
localStore, err := NewTestLocalStoreForAddr(datadir, addr.Over())
params := NewDefaultLocalStoreParams()
params.Init(datadir)
params.BaseKey = addr.Over()
localStore, err := NewTestLocalStoreForAddr(params)
if err != nil {
t.Fatal(err)
}

292
swarm/storage/resource.go
View file

@ -1,6 +1,7 @@
package storage
import (
"bytes"
"context"
"encoding/binary"
"errors"
@ -13,6 +14,7 @@ import (
"golang.org/x/net/idna"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/contracts/ens"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
@ -20,11 +22,12 @@ import (
const (
signatureLength = 65
indexSize = 16
indexSize = 18
DbDirName = "resource"
chunkSize = 4096 // temporary until we implement DPA in the resourcehandler
defaultStoreTimeout = 4000 * time.Millisecond
hasherCount = 8
resourceHash = SHA3Hash
)
type ResourceError struct {
@ -61,10 +64,6 @@ type ResourceLookupParams struct {
Max uint32
}
type SignFunc func(common.Hash) (Signature, error)
type nameHashFunc func(string) common.Hash
// Encapsulates an specific resource update. When synced it contains the most recent
// version of the resource update data.
type resource struct {
@ -88,15 +87,6 @@ func (self *resource) NameHash() common.Hash {
return self.nameHash
}
// Implement to activate validation of resource updates
// Specifically signing data and verification of signatures
type ResourceValidator interface {
hashSize() int
checkAccess(string, common.Address) (bool, error)
NameHash(string) common.Hash // nameHashFunc
sign(common.Hash) (Signature, error) // SignFunc
}
type headerGetter interface {
HeaderByNumber(context.Context, string, *big.Int) (*types.Header, error)
}
@ -146,90 +136,123 @@ type headerGetter interface {
// headerlength|period|version|identifier|data
//
// if a validator is active, the chunk data is:
// sign(resourcedata)|resourcedata
// resourcedata|sign(resourcedata)
// otherwise, the chunk data is the same as the resourcedata
//
// headerlength is a 16 bit value containing the byte length of period|version|name
// period and version are both 32 bit values. name can have arbitrary length
//
// NOTE: the following is yet to be implemented
// The resource update chunks will be stored in the swarm, but receive special
// treatment as their keys do not validate as hashes of their data. They are also
// stored using a separate store, and forwarding/syncing protocols carry per-chunk
// flags to tell whether the chunk can be validated or not; if not it is to be
// treated as a resource update chunk.
//
// TODO: Include modtime in chunk data + signature
type ResourceHandler struct {
ChunkStore
validator ResourceValidator
chunkStore ChunkStore
HashSize int
signer ResourceSigner
ethClient headerGetter
ensClient *ens.ENS
resources map[string]*resource
hashPool sync.Pool
resourceLock sync.RWMutex
nameHash nameHashFunc
storeTimeout time.Duration
queryMaxPeriods *ResourceLookupParams
}
type ResourceHandlerParams struct {
Validator ResourceValidator
QueryMaxPeriods *ResourceLookupParams
Signer ResourceSigner
EthClient headerGetter
EnsClient *ens.ENS
}
// Create or open resource update chunk store
func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, params *ResourceHandlerParams) (*ResourceHandler, error) {
func NewResourceHandler(params *ResourceHandlerParams) (*ResourceHandler, error) {
if params.QueryMaxPeriods == nil {
params.QueryMaxPeriods = &ResourceLookupParams{
Limit: false,
}
}
rh := &ResourceHandler{
ChunkStore: chunkStore,
ethClient: ethClient,
ethClient: params.EthClient,
ensClient: params.EnsClient,
resources: make(map[string]*resource),
validator: params.Validator,
storeTimeout: defaultStoreTimeout,
signer: params.Signer,
hashPool: sync.Pool{
New: func() interface{} {
return MakeHashFunc(SHA3Hash)()
return MakeHashFunc(resourceHash)()
},
},
queryMaxPeriods: params.QueryMaxPeriods,
}
if rh.validator != nil {
rh.nameHash = rh.validator.NameHash
} else {
rh.nameHash = func(name string) common.Hash {
hasher := rh.hashPool.Get().(SwarmHash)
defer rh.hashPool.Put(hasher)
hasher.Reset()
hasher.Write([]byte(name))
hashval := common.BytesToHash(hasher.Sum(nil))
log.Debug("generic namehasher", "name", name, "hash", hashval)
return hashval
}
}
for i := 0; i < hasherCount; i++ {
hashfunc := MakeHashFunc(SHA3Hash)()
hashfunc := MakeHashFunc(resourceHash)()
if rh.HashSize == 0 {
rh.HashSize = hashfunc.Size()
}
rh.hashPool.Put(hashfunc)
}
return rh, nil
}
func (self *ResourceHandler) IsValidated() bool {
return self.validator == nil
// Sets the store backend for resource updates
func (self *ResourceHandler) SetStore(store ChunkStore) {
self.chunkStore = store
}
func (self *ResourceHandler) HashSize() int {
return self.validator.hashSize()
// Chunk Validation method (matches ChunkValidatorFunc signature)
//
// If resource update, owner is checked against ENS record of resource name inferred from chunk data
// If parsed signature is nil, validates automatically
// If not resource update, it validates are root chunk if length is indexSize and first two bytes are 0
func (self *ResourceHandler) Validate(key Key, data []byte) bool {
signature, period, version, name, parseddata, err := self.parseUpdate(data)
if err != nil {
if len(data) == indexSize {
if bytes.Equal(data[:2], []byte{0, 0}) {
return true
}
}
return false
} else if signature == nil {
if !bytes.Equal(self.resourceHash(period, version, ens.EnsNode(name)), key) {
return false
}
return true
}
digest := self.keyDataHash(key, parseddata)
addr, err := getAddressFromDataSig(digest, *signature)
if err != nil {
return false
}
ok, _ := self.checkAccess(name, addr)
return ok
}
// If no ens client is supplied, resource updates are not validated
func (self *ResourceHandler) IsValidated() bool {
return self.ensClient != nil
}
// Create the resource update digest used in signatures
func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash {
hasher := self.hashPool.Get().(SwarmHash)
defer self.hashPool.Put(hasher)
hasher.Reset()
hasher.Write(key[:])
hasher.Write(data)
return common.BytesToHash(hasher.Sum(nil))
}
// Checks if current address matches owner address of ENS
func (self *ResourceHandler) checkAccess(name string, address common.Address) (bool, error) {
if self.ensClient == nil {
return true, nil
}
owneraddress, err := self.ensClient.Owner(ens.EnsNode(name))
return (address == owneraddress), err
}
// get data from current resource
func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) {
rsrc := self.getResource(name)
if rsrc == nil || !rsrc.isSynced() {
@ -278,10 +301,10 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
return nil, NewResourceError(ErrInvalidValue, fmt.Sprintf("Invalid name: '%s'", name))
}
nameHash := self.nameHash(name)
nameHash := ens.EnsNode(name)
if self.validator != nil {
signature, err := self.validator.sign(nameHash)
if self.signer != nil {
signature, err := self.signer.Sign(nameHash)
if err != nil {
return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err))
}
@ -289,7 +312,7 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
if err != nil {
return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Retrieve address from signature fail: %v", err))
}
ok, err := self.validator.checkAccess(name, addr)
ok, err := self.checkAccess(name, addr)
if err != nil {
return nil, err
} else if !ok {
@ -305,15 +328,17 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
// chunk with key equal to namehash points to data of first blockheight + update frequency
// from this we know from what blockheight we should look for updates, and how often
chunk := NewChunk(Key(nameHash.Bytes()), nil)
key := Key(nameHash.Bytes())
chunk := NewChunk(key, nil)
chunk.SData = make([]byte, indexSize)
// root block has first two bytes both, which distinguishes from update bytes
val := make([]byte, 8)
binary.LittleEndian.PutUint64(val, currentblock)
copy(chunk.SData[:8], val)
copy(chunk.SData[2:10], val)
binary.LittleEndian.PutUint64(val, frequency)
copy(chunk.SData[8:], val)
self.Put(chunk)
copy(chunk.SData[10:], val)
self.chunkStore.Put(chunk)
log.Debug("new resource", "name", name, "key", nameHash, "startBlock", currentblock, "frequency", frequency)
rsrc := &resource{
@ -336,12 +361,8 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
// root chunk.
// It is the callers responsibility to make sure that this chunk exists (if the resource
// update root data was retrieved externally, it typically doesn't)
//
// If maxPeriod is -1, the default QueryMaxPeriod from ResourceHandlerParams will be used
// if maxPeriod is 0, there will be no limit on period hops
// if maxPeriod > 0, the given value will be the limit of period hops
func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh, maxLookup)
return self.LookupVersion(ctx, ens.EnsNode(name), name, period, version, refresh, maxLookup)
}
func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@ -361,7 +382,7 @@ func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.
//
// See also (*ResourceHandler).LookupVersion
func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh, maxLookup)
return self.LookupHistorical(ctx, ens.EnsNode(name), name, period, refresh, maxLookup)
}
func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@ -383,7 +404,7 @@ func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash comm
//
// See also (*ResourceHandler).LookupHistorical
func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
return self.LookupLatest(ctx, self.nameHash(name), name, refresh, maxLookup)
return self.LookupLatest(ctx, ens.EnsNode(name), name, refresh, maxLookup)
}
func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
@ -404,6 +425,10 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H
// base code for public lookup methods
func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool, maxLookup *ResourceLookupParams) (*resource, error) {
if self.chunkStore == nil {
return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before performing lookups")
}
if period == 0 {
return nil, NewResourceError(ErrInvalidValue, "period must be >0")
}
@ -426,7 +451,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
return nil, NewResourceError(ErrPeriodDepth, fmt.Sprintf("Lookup exceeded max period hops (%d)", maxLookup.Max))
}
key := self.resourceHash(period, version, rsrc.nameHash)
chunk, err := self.Get(key)
chunk, err := self.chunkStore.Get(key)
if err == nil {
if specificversion {
return self.updateResourceIndex(rsrc, chunk)
@ -436,7 +461,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
for {
newversion := version + 1
key := self.resourceHash(period, newversion, rsrc.nameHash)
newchunk, err := self.Get(key)
newchunk, err := self.chunkStore.Get(key)
if err != nil {
return self.updateResourceIndex(rsrc, chunk)
}
@ -472,7 +497,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref
rsrc.nameHash = nameHash
// get the root info chunk and update the cached value
chunk, err := self.Get(Key(rsrc.nameHash[:]))
chunk, err := self.chunkStore.Get(Key(rsrc.nameHash[:]))
if err != nil {
return nil, err
}
@ -481,8 +506,8 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref
if len(chunk.SData) != indexSize {
return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize))
}
rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8])
rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:])
rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[2:10])
rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[10:])
} else {
rsrc.name = self.resources[name].name
rsrc.nameHash = self.resources[name].nameHash
@ -501,8 +526,8 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) (
return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name))
}
log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version)
// only check signature if validator is present
if self.validator != nil {
// check signature (if signer algorithm is present)
if signature != nil {
digest := self.keyDataHash(chunk.Key, data)
_, err = getAddressFromDataSig(digest, *signature)
if err != nil {
@ -525,11 +550,16 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) (
// retrieve update metadata from chunk data
// mirrors newUpdateChunk()
func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) {
// 14 = header + one byte of name + one byte of data
if len(chunkdata) < 14 {
return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, "chunk less than 13 bytes cannot be a resource update chunk")
}
cursor := 0
headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2])
cursor += 2
datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2])
if int(headerlength+datalength+4) > len(chunkdata) {
exclsignlength := int(headerlength + datalength + 4)
if exclsignlength > len(chunkdata) || exclsignlength < 14 {
return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)))
}
var period uint32
@ -560,10 +590,13 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32,
// omit signatures if we have no validator
var signature *Signature
if self.validator != nil {
cursor += intdatalength
if self.signer != nil {
sigdata := chunkdata[cursor : cursor+signatureLength]
if len(sigdata) > 0 {
signature = &Signature{}
copy(signature[:], chunkdata[cursor:cursor+signatureLength])
copy(signature[:], sigdata)
}
}
return signature, period, version, name, data, nil
@ -589,8 +622,11 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt
func (self *ResourceHandler) update(ctx context.Context, name string, data []byte, multihash bool) (Key, error) {
if self.chunkStore == nil {
return nil, NewResourceError(ErrInit, "Call ResourceHandler.SetStore() before updating")
}
var signaturelength int
if self.validator != nil {
if self.signer != nil {
signaturelength = signatureLength
}
@ -628,10 +664,10 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
key := self.resourceHash(nextperiod, version, rsrc.nameHash)
var signature *Signature
if self.validator != nil {
if self.signer != nil {
// sign the data hash with the key
digest := self.keyDataHash(key, data)
sig, err := self.validator.sign(digest)
sig, err := self.signer.Sign(digest)
if err != nil {
return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err))
}
@ -642,15 +678,16 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
if err != nil {
return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Invalid data/signature: %v", err))
}
if self.signer != nil {
// check if the signer has access to update
ok, err := self.validator.checkAccess(name, addr)
ok, err := self.checkAccess(name, addr)
if err != nil {
return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err))
} else if !ok {
return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name))
}
}
}
var datalength int
if !multihash {
@ -659,7 +696,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
chunk := newUpdateChunk(key, signature, nextperiod, version, name, data, datalength)
// send the chunk
self.Put(chunk)
self.chunkStore.Put(chunk)
timeout := time.NewTimer(self.storeTimeout)
select {
case <-chunk.dbStoredC:
@ -679,7 +716,7 @@ func (self *ResourceHandler) update(ctx context.Context, name string, data []byt
// Closes the datastore.
// Always call this at shutdown to avoid data corruption.
func (self *ResourceHandler) Close() {
self.ChunkStore.Close()
self.chunkStore.Close()
}
func (self *ResourceHandler) getBlock(ctx context.Context, name string) (uint64, error) {
@ -789,52 +826,6 @@ func newUpdateChunk(key Key, signature *Signature, period uint32, version uint32
return chunk
}
// \TODO chunkSize is a workaround until the ChunkStore interface exports a method to get the chunk size directly
type resourceChunkStore struct {
localStore ChunkStore
netStore ChunkStore
chunkSize int64
}
func NewResourceChunkStore(localStore *LocalStore, request func(*Chunk) error) ChunkStore {
return &resourceChunkStore{
localStore: localStore,
netStore: NewNetStore(localStore, request),
}
}
func (r *resourceChunkStore) Get(key Key) (*Chunk, error) {
chunk, err := r.netStore.Get(key)
if err != nil {
return nil, err
}
// if the chunk has to be remotely retrieved, we define a timeout of how long to wait for it before failing.
// sadly due to the nature of swarm, the error will never be conclusive as to whether it was a network issue
// that caused the failure or that the chunk doesn't exist.
if chunk.ReqC == nil {
return chunk, nil
}
t := time.NewTimer(time.Second * 1)
select {
case <-t.C:
log.Trace("Timeout on resource chunk store")
return nil, fmt.Errorf("timeout")
case <-chunk.C:
log.Trace("Received resource update chunk")
}
return chunk, nil
}
func (r *resourceChunkStore) Put(chunk *Chunk) {
r.netStore.Put(chunk)
chunk.WaitToStore()
}
func (r *resourceChunkStore) Close() {
r.netStore.Close()
r.localStore.Close()
}
func getNextPeriod(start uint64, current uint64, frequency uint64) uint32 {
blockdiff := current - start
period := blockdiff / frequency
@ -857,16 +848,6 @@ func isSafeName(name string) bool {
return validname == name
}
// convenience for creating signature hashes of update data
func (self *ResourceHandler) keyDataHash(key Key, data []byte) common.Hash {
hasher := self.hashPool.Get().(SwarmHash)
defer self.hashPool.Put(hasher)
hasher.Reset()
hasher.Write(key[:])
hasher.Write(data)
return common.BytesToHash(hasher.Sum(nil))
}
// if first byte is the start of a multihash this function will try to parse it
// if successful it returns the length of multihash data, 0 otherwise
func isMultihash(data []byte) int {
@ -892,29 +873,20 @@ func isMultihash(data []byte) int {
return cursor + inthashlength
}
// TODO: this should not be part of production code, but currently swarm/testutil/http.go needs it
func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator, maxLimit *ResourceLookupParams) (*ResourceHandler, error) {
// TODO: this should not be exposed, but swarm/testutil/http.go needs it
func NewTestResourceHandler(datadir string, params *ResourceHandlerParams) (*ResourceHandler, error) {
path := filepath.Join(datadir, DbDirName)
basekey := make([]byte, 32)
hasher := MakeHashFunc(SHA3Hash)
dbStore, err := NewLDBStore(path, hasher, defaultLDBCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
dbStore.SetTrusted()
rh, err := NewResourceHandler(params)
if err != nil {
return nil, err
}
localStore := &LocalStore{
memStore: NewMemStore(dbStore, defaultCacheCapacity, defaultChunkRequestsCacheCapacity),
DbStore: dbStore,
localstoreparams := NewDefaultLocalStoreParams()
localstoreparams.Init(path)
localStore, err := NewLocalStore(localstoreparams, nil)
if err != nil {
return nil, err
}
resourceChunkStore := NewResourceChunkStore(localStore, nil)
if maxLimit == nil {
maxLimit = &ResourceLookupParams{
Limit: false,
}
}
params := &ResourceHandlerParams{
Validator: validator,
QueryMaxPeriods: maxLimit,
}
return NewResourceHandler(hasher, resourceChunkStore, ethClient, params)
localStore.Validators = append(localStore.Validators, rh)
rh.SetStore(localStore)
return rh, nil
}

52
swarm/storage/resource_ens.go
View file

@ -1,52 +0,0 @@
package storage
import (
"errors"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/contracts/ens"
)
type baseValidator struct {
signFunc SignFunc
hashsize int
}
func (b *baseValidator) sign(datahash common.Hash) (signature Signature, err error) {
if b.signFunc == nil {
return signature, errors.New("No signature function")
}
return b.signFunc(datahash)
}
func (b *baseValidator) hashSize() int {
return b.hashsize
}
type OwnerValidator interface {
ValidateOwner(name string, address common.Address) (bool, error)
}
// ENS validation of mutable resource owners
type ENSValidator struct {
*baseValidator
api OwnerValidator
}
func NewENSValidator(contractaddress common.Address, ownerValidator OwnerValidator, signFunc SignFunc) *ENSValidator {
return &ENSValidator{
baseValidator: &baseValidator{
signFunc: signFunc,
hashsize: common.HashLength,
},
api: ownerValidator,
}
}
func (self *ENSValidator) checkAccess(name string, address common.Address) (bool, error) {
return self.api.ValidateOwner(name, address)
}
func (self *ENSValidator) NameHash(name string) common.Hash {
return ens.EnsNode(name)
}

16
swarm/storage/resource_sign.go
View file

@ -7,14 +7,20 @@ import (
"github.com/ethereum/go-ethereum/crypto"
)
// matches the SignFunc type
func NewGenericResourceSigner(privKey *ecdsa.PrivateKey) SignFunc {
return func(data common.Hash) (signature Signature, err error) {
signaturebytes, err := crypto.Sign(data.Bytes(), privKey)
// Signs resource updates
type ResourceSigner interface {
Sign(common.Hash) (Signature, error)
}
type GenericResourceSigner struct {
PrivKey *ecdsa.PrivateKey
}
func (self *GenericResourceSigner) Sign(data common.Hash) (signature Signature, err error) {
signaturebytes, err := crypto.Sign(data.Bytes(), self.PrivKey)
if err != nil {
return
}
copy(signature[:], signaturebytes)
return
}
}

214
swarm/storage/resource_test.go
View file

@ -3,7 +3,6 @@ package storage
import (
"bytes"
"context"
"crypto/ecdsa"
"crypto/rand"
"encoding/binary"
"flag"
@ -83,14 +82,14 @@ func TestResourceReverse(t *testing.T) {
}
// set up rpc and create resourcehandler
rh, _, _, teardownTest, err := setupTest(nil, newTestValidator(signer.signContent))
rh, _, teardownTest, err := setupTest(nil, nil, signer)
if err != nil {
t.Fatal(err)
}
defer teardownTest()
// generate a hash for block 4200 version 1
key := rh.resourceHash(period, version, rh.nameHash(safeName))
key := rh.resourceHash(period, version, ens.EnsNode(safeName))
// generate some bogus data for the chunk and sign it
data := make([]byte, 8)
@ -101,7 +100,7 @@ func TestResourceReverse(t *testing.T) {
testHasher.Reset()
testHasher.Write(data)
digest := rh.keyDataHash(key, data)
sig, err := rh.validator.sign(digest)
sig, err := rh.signer.Sign(digest)
if err != nil {
t.Fatal(err)
}
@ -118,7 +117,7 @@ func TestResourceReverse(t *testing.T) {
if err != nil {
t.Fatalf("Retrieve address from signature fail: %v", err)
}
originaladdress := crypto.PubkeyToAddress(signer.privKey.PublicKey)
originaladdress := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
// check that the metadata retrieved from the chunk matches what we gave it
if recoveredaddress != originaladdress {
@ -149,7 +148,7 @@ func TestResourceHandler(t *testing.T) {
backend := &fakeBackend{
blocknumber: int64(startBlock),
}
rh, datadir, _, teardownTest, err := setupTest(backend, nil)
rh, datadir, teardownTest, err := setupTest(backend, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -164,15 +163,15 @@ func TestResourceHandler(t *testing.T) {
}
// check that the new resource is stored correctly
namehash := rh.nameHash(safeName)
chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(Key(namehash[:]))
namehash := ens.EnsNode(safeName)
chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(Key(namehash[:]))
if err != nil {
t.Fatal(err)
} else if len(chunk.SData) < 16 {
t.Fatalf("chunk data must be minimum 16 bytes, is %d", len(chunk.SData))
}
startblocknumber := binary.LittleEndian.Uint64(chunk.SData[:8])
chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[8:])
startblocknumber := binary.LittleEndian.Uint64(chunk.SData[2:10])
chunkfrequency := binary.LittleEndian.Uint64(chunk.SData[10:])
if startblocknumber != uint64(backend.blocknumber) {
t.Fatalf("stored block number %d does not match provided block number %d", startblocknumber, backend.blocknumber)
}
@ -219,10 +218,14 @@ func TestResourceHandler(t *testing.T) {
// it will match on second iteration startblocknumber + (resourceFrequency * 3)
fwdBlocks(int(resourceFrequency*2)-1, backend)
lookupParams := &ResourceLookupParams{
rhparams := &ResourceHandlerParams{
QueryMaxPeriods: &ResourceLookupParams{
Limit: false,
},
Signer: nil,
EthClient: rh.ethClient,
}
rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil, lookupParams)
rh2, err := NewTestResourceHandler(datadir, rhparams)
if err != nil {
t.Fatal(err)
}
@ -244,7 +247,7 @@ func TestResourceHandler(t *testing.T) {
log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
// specific block, latest version
rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, lookupParams)
rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, rh2.queryMaxPeriods)
if err != nil {
t.Fatal(err)
}
@ -255,7 +258,7 @@ func TestResourceHandler(t *testing.T) {
log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
// specific block, specific version
rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, lookupParams)
rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, rh2.queryMaxPeriods)
if err != nil {
t.Fatal(err)
}
@ -267,6 +270,65 @@ func TestResourceHandler(t *testing.T) {
}
// create ENS enabled resource update, with and without valid owner
func TestResourceENSOwner(t *testing.T) {
// signer containing private key
signer, err := newTestSigner()
if err != nil {
t.Fatal(err)
}
// ens address and transact options
addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
transactOpts := bind.NewKeyedTransactor(signer.PrivKey)
// set up ENS sim
domainparts := strings.Split(safeName, ".")
contractAddr, contractbackend, err := setupENS(addr, transactOpts, domainparts[0], domainparts[1])
if err != nil {
t.Fatal(err)
}
ensClient, err := ens.NewENS(transactOpts, contractAddr, contractbackend)
if err != nil {
t.Fatal(err)
}
// set up rpc and create resourcehandler with ENS sim backend
rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer)
if err != nil {
t.Fatal(err)
}
defer teardownTest()
// create new resource when we are owner = ok
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_, err = rh.NewResource(ctx, safeName, resourceFrequency)
if err != nil {
t.Fatalf("Create resource fail: %v", err)
}
data := []byte("foo")
// update resource when we are owner = ok
_, err = rh.Update(ctx, safeName, data)
if err != nil {
t.Fatalf("Update resource fail: %v", err)
}
// update resource when we are not owner = !ok
signertwo, err := newTestSigner()
if err != nil {
t.Fatal(err)
}
rh.signer = signertwo
_, err = rh.Update(ctx, safeName, data)
if err == nil {
t.Fatalf("Expected resource update fail due to owner mismatch")
}
}
func TestResourceMultihash(t *testing.T) {
// signer containing private key
@ -274,7 +336,6 @@ func TestResourceMultihash(t *testing.T) {
if err != nil {
t.Fatal(err)
}
validator := newTestValidator(signer.signContent)
// make fake backend, set up rpc and create resourcehandler
backend := &fakeBackend{
@ -282,7 +343,7 @@ func TestResourceMultihash(t *testing.T) {
}
// set up rpc and create resourcehandler
rh, datadir, _, teardownTest, err := setupTest(backend, nil)
rh, datadir, teardownTest, err := setupTest(backend, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -298,7 +359,7 @@ func TestResourceMultihash(t *testing.T) {
// we're naïvely assuming keccak256 for swarm hashes
// if it ever changes this test should also change
swarmhashbytes := rh.nameHash("foo")
swarmhashbytes := ens.EnsNode("foo")
swarmhashmulti, err := multihash.Encode(swarmhashbytes.Bytes(), multihash.KECCAK_256)
if err != nil {
t.Fatal(err)
@ -352,8 +413,16 @@ func TestResourceMultihash(t *testing.T) {
}
rh.Close()
rhparams := &ResourceHandlerParams{
QueryMaxPeriods: &ResourceLookupParams{
Limit: false,
},
Signer: signer,
EthClient: rh.ethClient,
EnsClient: rh.ensClient,
}
// test with signed data
rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator, nil)
rh2, err := NewTestResourceHandler(datadir, rhparams)
if err != nil {
t.Fatal(err)
}
@ -394,9 +463,7 @@ func TestResourceMultihash(t *testing.T) {
}
}
// create ENS enabled resource update, with and without valid owner
func TestResourceENSOwner(t *testing.T) {
func TestResourceChunkValidator(t *testing.T) {
// signer containing private key
signer, err := newTestSigner()
if err != nil {
@ -404,8 +471,8 @@ func TestResourceENSOwner(t *testing.T) {
}
// ens address and transact options
addr := crypto.PubkeyToAddress(signer.privKey.PublicKey)
transactOpts := bind.NewKeyedTransactor(signer.privKey)
addr := crypto.PubkeyToAddress(signer.PrivKey.PublicKey)
transactOpts := bind.NewKeyedTransactor(signer.PrivKey)
// set up ENS sim
domainparts := strings.Split(safeName, ".")
@ -418,10 +485,9 @@ func TestResourceENSOwner(t *testing.T) {
if err != nil {
t.Fatal(err)
}
validator := NewENSValidator(contractAddr, newTestResolver(ensClient), signer.signContent)
// set up rpc and create resourcehandler with ENS sim backend
rh, _, _, teardownTest, err := setupTest(contractbackend, validator)
rh, _, teardownTest, err := setupTest(contractbackend, ensClient, signer)
if err != nil {
t.Fatal(err)
}
@ -430,27 +496,21 @@ func TestResourceENSOwner(t *testing.T) {
// create new resource when we are owner = ok
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_, err = rh.NewResource(ctx, safeName, resourceFrequency)
rsrc, err := rh.NewResource(ctx, safeName, resourceFrequency)
if err != nil {
t.Fatalf("Create resource fail: %v", err)
}
data := []byte("foo")
// update resource when we are owner = ok
_, err = rh.Update(ctx, safeName, data)
key := rh.resourceHash(1, 1, rsrc.nameHash)
digest := rh.keyDataHash(key, data)
sig, err := rh.signer.Sign(digest)
if err != nil {
t.Fatalf("Update resource fail: %v", err)
t.Fatalf("sign fail: %v", err)
}
// update resource when we are owner = ok
signertwo, err := newTestSigner()
if err != nil {
t.Fatal(err)
}
rh.validator.(*ENSValidator).signFunc = signertwo.signContent
_, err = rh.Update(ctx, safeName, data)
if err == nil {
t.Fatalf("Expected resource update fail due to owner mismatch")
chunk := newUpdateChunk(key, &sig, 1, 1, safeName, data, len(data))
if !rh.Validate(chunk.Key, chunk.SData) {
t.Fatal("Chunk validator fail")
}
}
@ -462,7 +522,7 @@ func fwdBlocks(count int, backend *fakeBackend) {
}
// create rpc and resourcehandler
func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceHandler, datadir string, signer *testSigner, teardown func(), err error) {
func setupTest(backend headerGetter, ensBackend *ens.ENS, signer ResourceSigner) (rh *ResourceHandler, datadir string, teardown func(), err error) {
var fsClean func()
var rpcClean func()
@ -478,14 +538,22 @@ func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceH
// temp datadir
datadir, err = ioutil.TempDir("", "rh")
if err != nil {
return nil, "", nil, nil, err
return nil, "", nil, err
}
fsClean = func() {
os.RemoveAll(datadir)
}
rh, err = NewTestResourceHandler(datadir, backend, validator, &ResourceLookupParams{Limit: false})
return rh, datadir, signer, cleanF, nil
rhparams := &ResourceHandlerParams{
QueryMaxPeriods: &ResourceLookupParams{
Limit: false,
},
Signer: signer,
EthClient: backend,
EnsClient: ensBackend,
}
rh, err = NewTestResourceHandler(datadir, rhparams)
return rh, datadir, cleanF, nil
}
// Set up simulated ENS backend for use with ENSResourceHandler tests
@ -531,55 +599,18 @@ func setupENS(addr common.Address, transactOpts *bind.TransactOpts, sub string,
return contractAddress, contractBackend, nil
}
// implementation of an external signer to pass to validator
type testSigner struct {
privKey *ecdsa.PrivateKey
hasher SwarmHash
signContent SignFunc
}
func newTestSigner() (*testSigner, error) {
func newTestSigner() (*GenericResourceSigner, error) {
privKey, err := crypto.GenerateKey()
if err != nil {
return nil, err
}
return &testSigner{
privKey: privKey,
hasher: testHasher,
signContent: NewGenericResourceSigner(privKey),
return &GenericResourceSigner{
PrivKey: privKey,
}, nil
}
// Default fallthrough validation of mutable resource ownership
type testValidator struct {
*baseValidator
hashFunc func(string) common.Hash
}
func newTestValidator(signFunc SignFunc) *testValidator {
return &testValidator{
baseValidator: &baseValidator{
signFunc: signFunc,
},
hashFunc: func(name string) common.Hash {
testHasher.Reset()
testHasher.Write([]byte(name))
return common.BytesToHash(testHasher.Sum(nil))
},
}
}
func (self *testValidator) checkAccess(name string, address common.Address) (bool, error) {
return true, nil
}
func (self *testValidator) NameHash(name string) common.Hash {
return self.hashFunc(name)
}
func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) {
chunk, err := rh.ChunkStore.(*resourceChunkStore).localStore.(*LocalStore).memStore.Get(key)
chunk, err := rh.chunkStore.(*LocalStore).memStore.Get(key)
if err != nil {
return nil, err
}
@ -589,18 +620,3 @@ func getUpdateDirect(rh *ResourceHandler, key Key) ([]byte, error) {
}
return data, nil
}
type testResolver struct {
ens *ens.ENS
}
func newTestResolver(ens *ens.ENS) *testResolver {
return &testResolver{
ens: ens,
}
}
func (r *testResolver) ValidateOwner(name string, address common.Address) (bool, error) {
addr, err := r.ens.Owner(ens.EnsNode(name))
return addr == address, err
}

68
swarm/storage/types.go
View file

@ -29,6 +29,7 @@ import (
"github.com/ethereum/go-ethereum/bmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/log"
)
const MaxPO = 7
@ -180,18 +181,18 @@ type Chunk struct {
dbStoredC chan bool // never remove a chunk from memStore before it is written to dbStore
dbStored bool
dbStoredMu *sync.Mutex
errored bool // flag which is set when the chunk request has errored or timeouted
errored error // flag which is set when the chunk request has errored or timeouted
erroredMu sync.Mutex
}
func (c *Chunk) SetErrored(val bool) {
func (c *Chunk) SetErrored(err error) {
c.erroredMu.Lock()
defer c.erroredMu.Unlock()
c.errored = val
c.errored = err
}
func (c *Chunk) GetErrored() bool {
func (c *Chunk) GetErrored() error {
c.erroredMu.Lock()
defer c.erroredMu.Unlock()
@ -257,6 +258,34 @@ func (self *LazyTestSectionReader) Size(chan bool) (int64, error) {
return self.SectionReader.Size(), nil
}
type StoreParams struct {
Hash SwarmHasher `toml:"-"`
DbCapacity uint64
CacheCapacity uint
ChunkRequestsCacheCapacity uint
BaseKey []byte
}
func NewDefaultStoreParams() *StoreParams {
return NewStoreParams(defaultLDBCapacity, defaultCacheCapacity, defaultChunkRequestsCacheCapacity, nil, nil)
}
func NewStoreParams(ldbCap uint64, cacheCap uint, requestsCap uint, hash SwarmHasher, basekey []byte) *StoreParams {
if basekey == nil {
basekey = make([]byte, 32)
}
if hash == nil {
hash = MakeHashFunc("SHA3")
}
return &StoreParams{
Hash: hash,
DbCapacity: ldbCap,
CacheCapacity: cacheCap,
ChunkRequestsCacheCapacity: requestsCap,
BaseKey: basekey,
}
}
type ChunkData []byte
type Reference []byte
@ -285,3 +314,34 @@ func (c ChunkData) Size() int64 {
func (c ChunkData) Data() []byte {
return c[8:]
}
type ChunkValidator interface {
Validate(key Key, data []byte) bool
}
// Provides method for validation of content address in chunks
// Holds the corresponding hasher to create the address
type ContentAddressValidator struct {
Hasher SwarmHasher
}
// Constructor
func NewContentAddressValidator(hasher SwarmHasher) *ContentAddressValidator {
return &ContentAddressValidator{
Hasher: hasher,
}
}
// Validate that the given key is a valid content address for the given data
func (self *ContentAddressValidator) Validate(key Key, data []byte) bool {
hasher := self.Hasher()
hasher.ResetWithLength(data[:8])
hasher.Write(data[8:])
hash := hasher.Sum(nil)
if !bytes.Equal(hash, key[:]) {
log.Error("invalid content address", "expected", fmt.Sprintf("%x", hash), "have", key)
return false
}
return true
}

96
swarm/swarm.go
View file

@ -117,15 +117,6 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
}
log.Debug(fmt.Sprintf("Setting up Swarm service components"))
hash := storage.MakeHashFunc(config.DPAParams.Hash)
self.lstore, err = storage.NewLocalStore(hash, config.StoreParams, common.FromHex(config.BzzKey), mockStore)
if err != nil {
return
}
// setup local store
log.Debug(fmt.Sprintf("Set up local storage"))
kp := network.NewKadParams()
to := network.NewKademlia(
common.FromHex(config.BzzKey),
@ -146,40 +137,16 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
HiveParams: config.HiveParams,
}
db := storage.NewDBAPI(self.lstore)
delivery := stream.NewDelivery(to, db)
// TODO: decide on intervals store file location
stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db"))
if err != nil {
return
}
self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{
DoSync: true,
DoRetrieve: true,
SyncUpdateDelay: config.SyncUpdateDelay,
})
self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)
// set up DPA, the cloud storage local access layer
dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams)
log.Debug(fmt.Sprintf("-> Content Store API"))
// Pss = postal service over swarm (devp2p over bzz)
if self.config.PssEnabled {
pssparams := pss.NewPssParams(self.privateKey)
self.ps = pss.NewPss(to, pssparams)
if pss.IsActiveHandshake {
pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
}
}
// set up high level api
//transactOpts := bind.NewKeyedTransactor(self.privateKey)
var resolver *api.MultiResolver
var ensresolver *ens.ENS
if len(config.EnsAPIs) > 0 {
opts := []api.MultiResolverOption{}
for _, c := range config.EnsAPIs {
@ -188,6 +155,7 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
if err != nil {
return nil, err
}
ensresolver = r.ENS
opts = append(opts, api.MultiResolverOptionWithResolver(r, tld))
}
@ -195,19 +163,67 @@ func NewSwarm(ctx *node.ServiceContext, backend chequebook.Backend, config *api.
self.dns = resolver
}
self.lstore, err = storage.NewLocalStore(config.LocalStoreParams, mockStore)
if err != nil {
return
}
var resourceHandler *storage.ResourceHandler
// if use resource updates
if self.config.ResourceEnabled && resolver != nil {
resourceparams := &storage.ResourceHandlerParams{
Validator: storage.NewENSValidator(config.EnsRoot, resolver, storage.NewGenericResourceSigner(self.privateKey)),
resolver.SetNameHash(ens.EnsNode)
rhparams := &storage.ResourceHandlerParams{
// TODO: config parameter to set limits
QueryMaxPeriods: &storage.ResourceLookupParams{
Limit: false,
},
Signer: &storage.GenericResourceSigner{
PrivKey: self.privateKey,
},
EthClient: resolver,
EnsClient: ensresolver,
}
resolver.SetNameHash(resourceparams.Validator.NameHash)
hashfunc := storage.MakeHashFunc(storage.SHA3Hash)
chunkStore := storage.NewResourceChunkStore(self.lstore, func(*storage.Chunk) error { return nil })
resourceHandler, err = storage.NewResourceHandler(hashfunc, chunkStore, resolver, resourceparams)
resourceHandler, err = storage.NewResourceHandler(rhparams)
if err != nil {
return nil, err
}
resourceHandler.SetStore(self.lstore)
}
var validators []storage.ChunkValidator
validators = append(validators, storage.NewContentAddressValidator(storage.MakeHashFunc(storage.SHA3Hash)))
if resourceHandler != nil {
validators = append(validators, resourceHandler)
}
self.lstore.Validators = validators
// setup local store
log.Debug(fmt.Sprintf("Set up local storage"))
db := storage.NewDBAPI(self.lstore)
delivery := stream.NewDelivery(to, db)
self.streamer = stream.NewRegistry(addr, delivery, db, stateStore, &stream.RegistryOptions{
DoSync: true,
DoRetrieve: true,
SyncUpdateDelay: config.SyncUpdateDelay,
})
// set up DPA, the cloud storage local access layer
dpaChunkStore := storage.NewNetStore(self.lstore, self.streamer.Retrieve)
log.Debug(fmt.Sprintf("-> Local Access to Swarm"))
// Swarm Hash Merklised Chunking for Arbitrary-length Document/File storage
self.dpa = storage.NewDPA(dpaChunkStore, self.config.DPAParams)
log.Debug(fmt.Sprintf("-> Content Store API"))
self.bzz = network.NewBzz(bzzconfig, to, stateStore, stream.Spec, self.streamer.Run)
// Pss = postal service over swarm (devp2p over bzz)
pssparams := pss.NewPssParams(self.privateKey)
self.ps = pss.NewPss(to, pssparams)
if pss.IsActiveHandshake {
pss.SetHandshakeController(self.ps, pss.NewHandshakeParams())
}
self.api = api.NewApi(self.dpa, self.dns, resourceHandler)

18
swarm/testutil/http.go
View file

@ -47,12 +47,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
if err != nil {
t.Fatal(err)
}
storeparams := &storage.StoreParams{
ChunkDbPath: dir,
DbCapacity: 5000000,
CacheCapacity: 5000,
}
localStore, err := storage.NewLocalStore(storage.MakeHashFunc(storage.SHA3Hash), storeparams, make([]byte, 32), nil)
storeparams := storage.NewDefaultLocalStoreParams()
storeparams.DbCapacity = 5000000
storeparams.CacheCapacity = 5000
storeparams.Init(dir)
localStore, err := storage.NewLocalStore(storeparams, nil)
if err != nil {
os.RemoveAll(dir)
t.Fatal(err)
@ -64,8 +63,11 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
if err != nil {
t.Fatal(err)
}
rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil, &storage.ResourceLookupParams{Limit: false})
rhparams := &storage.ResourceHandlerParams{
QueryMaxPeriods: &storage.ResourceLookupParams{},
EthClient: &fakeBackend{},
}
rh, err := storage.NewTestResourceHandler(resourceDir, rhparams)
if err != nil {
t.Fatal(err)
}

12
vendor/github.com/naoina/toml/encode.go generated vendored
View file

@ -61,20 +61,26 @@ func (cfg *Config) NewEncoder(w io.Writer) *Encoder {
// Encode writes the TOML of v to the stream.
// See the documentation for Marshal for details about the conversion of Go values to TOML.
func (e *Encoder) Encode(v interface{}) error {
rv := reflect.ValueOf(v)
var (
buf = &tableBuf{typ: ast.TableTypeNormal}
rv = reflect.ValueOf(v)
err error
)
for rv.Kind() == reflect.Ptr {
if rv.IsNil() {
return &marshalNilError{rv.Type()}
}
rv = rv.Elem()
}
buf := &tableBuf{typ: ast.TableTypeNormal}
var err error
switch rv.Kind() {
case reflect.Struct:
err = buf.structFields(e.cfg, rv)
case reflect.Map:
err = buf.mapFields(e.cfg, rv)
case reflect.Interface:
return e.Encode(rv.Interface())
default:
err = &marshalTableError{rv.Type()}
}

52
vendor/github.com/naoina/toml/parse.go generated vendored
View file

@ -97,6 +97,7 @@ type toml struct {
currentTable *ast.Table
s string
key string
tableKeys []string
val ast.Value
arr *array
stack []*stack
@ -180,12 +181,12 @@ func (p *tomlParser) SetArray(begin, end int) {
}
func (p *toml) SetTable(buf []rune, begin, end int) {
p.setTable(p.table, buf, begin, end)
rawName := string(buf[begin:end])
p.setTable(p.table, rawName, p.tableKeys)
p.tableKeys = nil
}
func (p *toml) setTable(parent *ast.Table, buf []rune, begin, end int) {
name := string(buf[begin:end])
names := splitTableKey(name)
func (p *toml) setTable(parent *ast.Table, name string, names []string) {
parent, err := p.lookupTable(parent, names[:len(names)-1])
if err != nil {
p.Error(err)
@ -230,12 +231,12 @@ func (p *tomlParser) SetTableString(begin, end int) {
}
func (p *toml) SetArrayTable(buf []rune, begin, end int) {
p.setArrayTable(p.table, buf, begin, end)
rawName := string(buf[begin:end])
p.setArrayTable(p.table, rawName, p.tableKeys)
p.tableKeys = nil
}
func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
name := string(buf[begin:end])
names := splitTableKey(name)
func (p *toml) setArrayTable(parent *ast.Table, name string, names []string) {
parent, err := p.lookupTable(parent, names[:len(names)-1])
if err != nil {
p.Error(err)
@ -260,11 +261,11 @@ func (p *toml) setArrayTable(parent *ast.Table, buf []rune, begin, end int) {
func (p *toml) StartInlineTable() {
p.skip = false
p.stack = append(p.stack, &stack{p.key, p.currentTable})
buf := []rune(p.key)
names := []string{p.key}
if p.arr == nil {
p.setTable(p.currentTable, buf, 0, len(buf))
p.setTable(p.currentTable, names[0], names)
} else {
p.setArrayTable(p.currentTable, buf, 0, len(buf))
p.setArrayTable(p.currentTable, names[0], names)
}
}
@ -282,6 +283,13 @@ func (p *toml) AddLineCount(i int) {
func (p *toml) SetKey(buf []rune, begin, end int) {
p.key = string(buf[begin:end])
if len(p.key) > 0 && p.key[0] == '"' {
p.key = p.unquote(p.key)
}
}
func (p *toml) AddTableKey() {
p.tableKeys = append(p.tableKeys, p.key)
}
func (p *toml) AddKeyValue() {
@ -352,25 +360,3 @@ func (p *toml) lookupTable(t *ast.Table, keys []string) (*ast.Table, error) {
}
return t, nil
}
func splitTableKey(tk string) []string {
key := make([]byte, 0, 1)
keys := make([]string, 0, 1)
inQuote := false
for i := 0; i < len(tk); i++ {
k := tk[i]
switch {
case k == tableSeparator && !inQuote:
keys = append(keys, string(key))
key = key[:0] // reuse buffer.
case k == '"':
inQuote = !inQuote
case (k == ' ' || k == '\t') && !inQuote:
// skip.
default:
key = append(key, k)
}
}
keys = append(keys, string(key))
return keys
}

8
vendor/github.com/naoina/toml/parse.peg generated vendored
View file

@ -29,7 +29,7 @@ key <- bareKey / quotedKey
bareKey <- <[0-9A-Za-z\-_]+> { p.SetKey(p.buffer, begin, end) }
quotedKey <- '"' <basicChar+> '"' { p.SetKey(p.buffer, begin-1, end+1) }
quotedKey <- < '"' basicChar* '"' > { p.SetKey(p.buffer, begin, end) }
val <- (
<datetime> { p.SetTime(begin, end) }
@ -55,7 +55,9 @@ inlineTable <- (
inlineTableKeyValues <- (keyval inlineTableValSep?)*
tableKey <- key (tableKeySep key)*
tableKey <- tableKeyComp (tableKeySep tableKeyComp)*
tableKeyComp <- key { p.AddTableKey() }
tableKeySep <- ws '.' ws
@ -117,7 +119,7 @@ timeNumoffset <- [\-+] timeHour ':' timeMinute
timeOffset <- 'Z' / timeNumoffset
partialTime <- timeHour ':' timeMinute ':' timeSecond timeSecfrac?
fullDate <- dateFullYear '-' dateMonth '-' dateMDay
fullTime <- partialTime timeOffset
fullTime <- partialTime timeOffset?
datetime <- (fullDate ('T' fullTime)?) / partialTime
digit <- [0-9]

1512
vendor/github.com/naoina/toml/parse.peg.go generated vendored
View file

File diff suppressed because it is too large Load diff

6
vendor/vendor.json vendored
View file

@ -292,10 +292,10 @@
"revisionTime": "2017-07-13T18:40:44Z"
},
{
"checksumSHA1": "2gmvVTDCks8cPhpmyDlvm0sbrXE=",
"checksumSHA1": "FYM/8R2CqS6PSNAoKl6X5gNJ20A=",
"path": "github.com/naoina/toml",
"revision": "ac014c6b6502388d89a85552b7208b8da7cfe104",
"revisionTime": "2017-04-10T21:57:17Z"
"revision": "9fafd69674167c06933b1787ae235618431ce87f",
"revisionTime": "2017-09-18T21:04:37Z"
},
{
"checksumSHA1": "xZBlSMT5o/A+EDOro6KbfHZwSNc=",