From d7f4f432409acde014f86f7f4c5ab209635d7bac Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 7 Mar 2018 16:20:27 +0100 Subject: [PATCH] swarm/storage: Add max period traversal for lookup query Add ResourceHandlerParams object --- swarm/storage/error.go | 1 + swarm/storage/ldbstore.go | 4 ++ swarm/storage/resource.go | 76 ++++++++++++++++++++++------------ swarm/storage/resource_test.go | 18 ++++---- swarm/testutil/http.go | 2 +- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/swarm/storage/error.go b/swarm/storage/error.go index 56e459731c..9a8676c8a3 100644 --- a/swarm/storage/error.go +++ b/swarm/storage/error.go @@ -9,5 +9,6 @@ const ( ErrNothingToReturn ErrInvalidSignature ErrNotSynced + ErrPeriodDepth ErrCnt ) diff --git a/swarm/storage/ldbstore.go b/swarm/storage/ldbstore.go index 8bc080c1ba..d1ea545aae 100644 --- a/swarm/storage/ldbstore.go +++ b/swarm/storage/ldbstore.go @@ -155,6 +155,10 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui return s, nil } +func (self *LDBStore) SetTrusted() { + self.trusted = true +} + // NewMockDbStore creates a new instance of DbStore with // mockStore set to a provided value. If mockStore argument is nil, // this function behaves exactly as NewDbStore. diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index f2567a762b..15bcf81e1e 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -48,7 +48,7 @@ func NewResourceError(code int, s string) error { err: s, } switch code { - case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced: + case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced, ErrPeriodDepth: r.code = code } return r @@ -157,28 +157,35 @@ type headerGetter interface { // TODO: Include modtime in chunk data + signature type ResourceHandler struct { ChunkStore - validator ResourceValidator - ethClient headerGetter - resources map[string]*resource - hashPool sync.Pool - resourceLock sync.RWMutex - nameHash nameHashFunc - storeTimeout time.Duration + validator ResourceValidator + ethClient headerGetter + resources map[string]*resource + hashPool sync.Pool + resourceLock sync.RWMutex + nameHash nameHashFunc + storeTimeout time.Duration + queryMaxPeriods uint +} + +type ResourceHandlerParams struct { + Validator ResourceValidator + QueryMaxPeriods uint } // Create or open resource update chunk store -func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, validator ResourceValidator) (*ResourceHandler, error) { +func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, params *ResourceHandlerParams) (*ResourceHandler, error) { rh := &ResourceHandler{ ChunkStore: chunkStore, ethClient: ethClient, resources: make(map[string]*resource), - validator: validator, + validator: params.Validator, storeTimeout: defaultStoreTimeout, hashPool: sync.Pool{ New: func() interface{} { return MakeHashFunc(SHA3Hash)() }, }, + queryMaxPeriods: params.QueryMaxPeriods, } if rh.validator != nil { @@ -320,17 +327,19 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ // It is the callers responsibility to make sure that this chunk exists (if the resource // update root data was retrieved externally, it typically doesn't) // -// -func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool) (*resource, error) { - return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh) +// If maxPeriod is -1, the default QueryMaxPeriod from ResourceHandlerParams will be used +// if maxPeriod is 0, there will be no limit on period hops +// if maxPeriod > 0, the given value will be the limit of period hops +func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) { + return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh, maxPeriod) } -func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) { rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } - return self.lookup(rsrc, period, version, refresh) + return self.lookup(rsrc, period, version, refresh, maxPeriod) } // Retrieves the latest version of the resource update identified by `name` @@ -341,16 +350,16 @@ func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common. // and returned. // // See also (*ResourceHandler).LookupVersion -func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool) (*resource, error) { - return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh) +func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool, maxPeriod int) (*resource, error) { + return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh, maxPeriod) } -func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool, maxPeriod int) (*resource, error) { rsrc, err := self.loadResource(nameHash, name, refresh) if err != nil { return nil, err } - return self.lookup(rsrc, period, 0, refresh) + return self.lookup(rsrc, period, 0, refresh, maxPeriod) } // Retrieves the latest version of the resource update identified by `name` @@ -363,11 +372,11 @@ func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash comm // Version iteration is done as in (*ResourceHandler).LookupHistorical // // See also (*ResourceHandler).LookupHistorical -func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool) (*resource, error) { - return self.LookupLatest(ctx, self.nameHash(name), name, refresh) +func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool, maxPeriod int) (*resource, error) { + return self.LookupLatest(ctx, self.nameHash(name), name, refresh, maxPeriod) } -func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool) (*resource, error) { +func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool, maxPeriod int) (*resource, error) { // get our blockheight at this time and the next block of the update period rsrc, err := self.loadResource(nameHash, name, refresh) @@ -379,11 +388,11 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H return nil, err } nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency) - return self.lookup(rsrc, nextperiod, 0, refresh) + return self.lookup(rsrc, nextperiod, 0, refresh, maxPeriod) } // base code for public lookup methods -func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { +func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) { if period == 0 { return nil, NewResourceError(ErrInvalidValue, "period must be >0") @@ -398,7 +407,14 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 version = 1 } + hops := 0 + if maxPeriod < 0 { + maxPeriod = int(self.queryMaxPeriods) + } for period > 0 { + if hops > maxPeriod && maxPeriod > 0 { + return nil, NewResourceError(ErrPeriodDepth, fmt.Sprintf("Lookup exceeded max period hops (%d)", maxPeriod)) + } key := self.resourceHash(period, version, rsrc.nameHash) chunk, err := self.Get(key) if err == nil { @@ -421,6 +437,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 } log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) period-- + hops++ } return nil, NewResourceError(ErrNotFound, "no updates found") } @@ -865,12 +882,13 @@ func isMultihash(data []byte) int { return cursor + inthashlength } -// TODO: this should not be exposed, but swarm/testutil/http.go needs it -func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator) (*ResourceHandler, error) { +// TODO: this should not be part of production code, but currently swarm/testutil/http.go needs it +func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator, maxPeriod uint) (*ResourceHandler, error) { path := filepath.Join(datadir, DbDirName) basekey := make([]byte, 32) hasher := MakeHashFunc(SHA3Hash) dbStore, err := NewLDBStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) + dbStore.SetTrusted() if err != nil { return nil, err } @@ -879,5 +897,9 @@ func NewTestResourceHandler(datadir string, ethClient headerGetter, validator Re DbStore: dbStore, } resourceChunkStore := NewResourceChunkStore(localStore, nil) - return NewResourceHandler(hasher, resourceChunkStore, ethClient, validator) + params := &ResourceHandlerParams{ + Validator: validator, + QueryMaxPeriods: maxPeriod, + } + return NewResourceHandler(hasher, resourceChunkStore, ethClient, params) } diff --git a/swarm/storage/resource_test.go b/swarm/storage/resource_test.go index dcd940829f..d4314f9ade 100644 --- a/swarm/storage/resource_test.go +++ b/swarm/storage/resource_test.go @@ -40,11 +40,9 @@ var ( func init() { var err error - verbose := flag.Bool("v", false, "verbose") + //loglevel := flag.Int("loglevel", 3, "loglevel") flag.Parse() - if *verbose { - log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) - } + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) safeName, err = ToSafeName(domainName) if err != nil { panic(err) @@ -222,8 +220,8 @@ func TestResourceHandler(t *testing.T) { // it will match on second iteration startblocknumber + (resourceFrequency * 3) fwdBlocks(int(resourceFrequency*2)-1, backend) - rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil) - _, err = rh2.LookupLatestByName(ctx, safeName, true) + rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil, 0) + _, err = rh2.LookupLatestByName(ctx, safeName, true, -1) if err != nil { t.Fatal(err) } @@ -241,7 +239,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, latest version - rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true) + rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, 0) if err != nil { t.Fatal(err) } @@ -252,7 +250,7 @@ func TestResourceHandler(t *testing.T) { log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) // specific block, specific version - rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true) + rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, 0) if err != nil { t.Fatal(err) } @@ -350,7 +348,7 @@ func TestResourceMultihash(t *testing.T) { rh.Close() // test with signed data - rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator) + rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator, 0) if err != nil { t.Fatal(err) } @@ -481,7 +479,7 @@ func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceH os.RemoveAll(datadir) } - rh, err = NewTestResourceHandler(datadir, backend, validator) + rh, err = NewTestResourceHandler(datadir, backend, validator, 0) return rh, datadir, signer, cleanF, nil } diff --git a/swarm/testutil/http.go b/swarm/testutil/http.go index 32866aec98..923d4a0085 100644 --- a/swarm/testutil/http.go +++ b/swarm/testutil/http.go @@ -70,7 +70,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer { t.Fatal(err) } - rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil) + rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil, -1) if err != nil { t.Fatal(err) }