forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-18 09:53:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

swarm/storage: Add max period traversal for lookup query

Browse source 
Add ResourceHandlerParams object
This commit is contained in:
lash 2018-03-07 16:20:27 +01:00
parent d5d0ebd350
commit d7f4f43240
5 changed files with 63 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
swarm/storage/error.go
View file

@ -9,5 +9,6 @@ const (
ErrNothingToReturn ErrNothingToReturn
ErrInvalidSignature ErrInvalidSignature
ErrNotSynced ErrNotSynced
ErrPeriodDepth
ErrCnt ErrCnt
) )

4
swarm/storage/ldbstore.go
View file

@ -155,6 +155,10 @@ func NewLDBStore(path string, hash SwarmHasher, capacity uint64, po func(Key) ui
return s, nil return s, nil
} }
func (self *LDBStore) SetTrusted() {
self.trusted = true
}
// NewMockDbStore creates a new instance of DbStore with // NewMockDbStore creates a new instance of DbStore with
// mockStore set to a provided value. If mockStore argument is nil, // mockStore set to a provided value. If mockStore argument is nil,
// this function behaves exactly as NewDbStore. // this function behaves exactly as NewDbStore.

76
swarm/storage/resource.go
View file

@ -48,7 +48,7 @@ func NewResourceError(code int, s string) error {
err: s, err: s,
} }
switch code { switch code {
case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced: case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced, ErrPeriodDepth:
r.code = code r.code = code
} }
return r return r
@ -157,28 +157,35 @@ type headerGetter interface {
// TODO: Include modtime in chunk data + signature // TODO: Include modtime in chunk data + signature
type ResourceHandler struct { type ResourceHandler struct {
ChunkStore ChunkStore
validator ResourceValidator validator ResourceValidator
ethClient headerGetter ethClient headerGetter
resources map[string]*resource resources map[string]*resource
hashPool sync.Pool hashPool sync.Pool
resourceLock sync.RWMutex resourceLock sync.RWMutex
nameHash nameHashFunc nameHash nameHashFunc
storeTimeout time.Duration storeTimeout time.Duration
queryMaxPeriods uint
}
type ResourceHandlerParams struct {
Validator ResourceValidator
QueryMaxPeriods uint
} }
// Create or open resource update chunk store // Create or open resource update chunk store
func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, validator ResourceValidator) (*ResourceHandler, error) { func NewResourceHandler(hasher SwarmHasher, chunkStore ChunkStore, ethClient headerGetter, params *ResourceHandlerParams) (*ResourceHandler, error) {
rh := &ResourceHandler{ rh := &ResourceHandler{
ChunkStore: chunkStore, ChunkStore: chunkStore,
ethClient: ethClient, ethClient: ethClient,
resources: make(map[string]*resource), resources: make(map[string]*resource),
validator: validator, validator: params.Validator,
storeTimeout: defaultStoreTimeout, storeTimeout: defaultStoreTimeout,
hashPool: sync.Pool{ hashPool: sync.Pool{
New: func() interface{} { New: func() interface{} {
return MakeHashFunc(SHA3Hash)() return MakeHashFunc(SHA3Hash)()
}, },
}, },
queryMaxPeriods: params.QueryMaxPeriods,
} }
if rh.validator != nil { if rh.validator != nil {
@ -320,17 +327,19 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ
// It is the callers responsibility to make sure that this chunk exists (if the resource // It is the callers responsibility to make sure that this chunk exists (if the resource
// update root data was retrieved externally, it typically doesn't) // update root data was retrieved externally, it typically doesn't)
// //
// // If maxPeriod is -1, the default QueryMaxPeriod from ResourceHandlerParams will be used
func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool) (*resource, error) { // if maxPeriod is 0, there will be no limit on period hops
return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh) // if maxPeriod > 0, the given value will be the limit of period hops
func (self *ResourceHandler) LookupVersionByName(ctx context.Context, name string, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) {
return self.LookupVersion(ctx, self.nameHash(name), name, period, version, refresh, maxPeriod)
} }
func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool) (*resource, error) { func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.Hash, name string, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) {
rsrc, err := self.loadResource(nameHash, name, refresh) rsrc, err := self.loadResource(nameHash, name, refresh)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return self.lookup(rsrc, period, version, refresh) return self.lookup(rsrc, period, version, refresh, maxPeriod)
} }
// Retrieves the latest version of the resource update identified by `name` // Retrieves the latest version of the resource update identified by `name`
@ -341,16 +350,16 @@ func (self *ResourceHandler) LookupVersion(ctx context.Context, nameHash common.
// and returned. // and returned.
// //
// See also (*ResourceHandler).LookupVersion // See also (*ResourceHandler).LookupVersion
func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool) (*resource, error) { func (self *ResourceHandler) LookupHistoricalByName(ctx context.Context, name string, period uint32, refresh bool, maxPeriod int) (*resource, error) {
return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh) return self.LookupHistorical(ctx, self.nameHash(name), name, period, refresh, maxPeriod)
} }
func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool) (*resource, error) { func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash common.Hash, name string, period uint32, refresh bool, maxPeriod int) (*resource, error) {
rsrc, err := self.loadResource(nameHash, name, refresh) rsrc, err := self.loadResource(nameHash, name, refresh)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return self.lookup(rsrc, period, 0, refresh) return self.lookup(rsrc, period, 0, refresh, maxPeriod)
} }
// Retrieves the latest version of the resource update identified by `name` // Retrieves the latest version of the resource update identified by `name`
@ -363,11 +372,11 @@ func (self *ResourceHandler) LookupHistorical(ctx context.Context, nameHash comm
// Version iteration is done as in (*ResourceHandler).LookupHistorical // Version iteration is done as in (*ResourceHandler).LookupHistorical
// //
// See also (*ResourceHandler).LookupHistorical // See also (*ResourceHandler).LookupHistorical
func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool) (*resource, error) { func (self *ResourceHandler) LookupLatestByName(ctx context.Context, name string, refresh bool, maxPeriod int) (*resource, error) {
return self.LookupLatest(ctx, self.nameHash(name), name, refresh) return self.LookupLatest(ctx, self.nameHash(name), name, refresh, maxPeriod)
} }
func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool) (*resource, error) { func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.Hash, name string, refresh bool, maxPeriod int) (*resource, error) {
// get our blockheight at this time and the next block of the update period // get our blockheight at this time and the next block of the update period
rsrc, err := self.loadResource(nameHash, name, refresh) rsrc, err := self.loadResource(nameHash, name, refresh)
@ -379,11 +388,11 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H
return nil, err return nil, err
} }
nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency) nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency)
return self.lookup(rsrc, nextperiod, 0, refresh) return self.lookup(rsrc, nextperiod, 0, refresh, maxPeriod)
} }
// base code for public lookup methods // base code for public lookup methods
func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool, maxPeriod int) (*resource, error) {
if period == 0 { if period == 0 {
return nil, NewResourceError(ErrInvalidValue, "period must be >0") return nil, NewResourceError(ErrInvalidValue, "period must be >0")
@ -398,7 +407,14 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
version = 1 version = 1
} }
hops := 0
if maxPeriod < 0 {
maxPeriod = int(self.queryMaxPeriods)
}
for period > 0 { for period > 0 {
if hops > maxPeriod && maxPeriod > 0 {
return nil, NewResourceError(ErrPeriodDepth, fmt.Sprintf("Lookup exceeded max period hops (%d)", maxPeriod))
}
key := self.resourceHash(period, version, rsrc.nameHash) key := self.resourceHash(period, version, rsrc.nameHash)
chunk, err := self.Get(key) chunk, err := self.Get(key)
if err == nil { if err == nil {
@ -421,6 +437,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3
} }
log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) log.Trace("rsrc update not found, checking previous period", "period", period, "key", key)
period-- period--
hops++
} }
return nil, NewResourceError(ErrNotFound, "no updates found") return nil, NewResourceError(ErrNotFound, "no updates found")
} }
@ -865,12 +882,13 @@ func isMultihash(data []byte) int {
return cursor + inthashlength return cursor + inthashlength
} }
// TODO: this should not be exposed, but swarm/testutil/http.go needs it // TODO: this should not be part of production code, but currently swarm/testutil/http.go needs it
func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator) (*ResourceHandler, error) { func NewTestResourceHandler(datadir string, ethClient headerGetter, validator ResourceValidator, maxPeriod uint) (*ResourceHandler, error) {
path := filepath.Join(datadir, DbDirName) path := filepath.Join(datadir, DbDirName)
basekey := make([]byte, 32) basekey := make([]byte, 32)
hasher := MakeHashFunc(SHA3Hash) hasher := MakeHashFunc(SHA3Hash)
dbStore, err := NewLDBStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) }) dbStore, err := NewLDBStore(path, hasher, singletonSwarmDbCapacity, func(k Key) (ret uint8) { return uint8(Proximity(basekey[:], k[:])) })
dbStore.SetTrusted()
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -879,5 +897,9 @@ func NewTestResourceHandler(datadir string, ethClient headerGetter, validator Re
DbStore: dbStore, DbStore: dbStore,
} }
resourceChunkStore := NewResourceChunkStore(localStore, nil) resourceChunkStore := NewResourceChunkStore(localStore, nil)
return NewResourceHandler(hasher, resourceChunkStore, ethClient, validator) params := &ResourceHandlerParams{
Validator: validator,
QueryMaxPeriods: maxPeriod,
}
return NewResourceHandler(hasher, resourceChunkStore, ethClient, params)
} }

18
swarm/storage/resource_test.go
View file

@ -40,11 +40,9 @@ var (
func init() { func init() {
var err error var err error
verbose := flag.Bool("v", false, "verbose") //loglevel := flag.Int("loglevel", 3, "loglevel")
flag.Parse() flag.Parse()
if *verbose { log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.Lvl(*loglevel), log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true)))))
}
safeName, err = ToSafeName(domainName) safeName, err = ToSafeName(domainName)
if err != nil { if err != nil {
panic(err) panic(err)
@ -222,8 +220,8 @@ func TestResourceHandler(t *testing.T) {
// it will match on second iteration startblocknumber + (resourceFrequency * 3) // it will match on second iteration startblocknumber + (resourceFrequency * 3)
fwdBlocks(int(resourceFrequency*2)-1, backend) fwdBlocks(int(resourceFrequency*2)-1, backend)
rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil) rh2, err := NewTestResourceHandler(datadir, rh.ethClient, nil, 0)
_, err = rh2.LookupLatestByName(ctx, safeName, true) _, err = rh2.LookupLatestByName(ctx, safeName, true, -1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -241,7 +239,7 @@ func TestResourceHandler(t *testing.T) {
log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) log.Debug("Latest lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
// specific block, latest version // specific block, latest version
rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true) rsrc, err := rh2.LookupHistoricalByName(ctx, safeName, 3, true, 0)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -252,7 +250,7 @@ func TestResourceHandler(t *testing.T) {
log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data) log.Debug("Historical lookup", "period", rh2.resources[safeName].lastPeriod, "version", rh2.resources[safeName].version, "data", rh2.resources[safeName].data)
// specific block, specific version // specific block, specific version
rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true) rsrc, err = rh2.LookupVersionByName(ctx, safeName, 3, 1, true, 0)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -350,7 +348,7 @@ func TestResourceMultihash(t *testing.T) {
rh.Close() rh.Close()
// test with signed data // test with signed data
rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator) rh2, err := NewTestResourceHandler(datadir, rh.ethClient, validator, 0)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -481,7 +479,7 @@ func setupTest(backend headerGetter, validator ResourceValidator) (rh *ResourceH
os.RemoveAll(datadir) os.RemoveAll(datadir)
} }
rh, err = NewTestResourceHandler(datadir, backend, validator) rh, err = NewTestResourceHandler(datadir, backend, validator, 0)
return rh, datadir, signer, cleanF, nil return rh, datadir, signer, cleanF, nil
} }

2
swarm/testutil/http.go
View file

@ -70,7 +70,7 @@ func NewTestSwarmServer(t *testing.T) *TestSwarmServer {
t.Fatal(err) t.Fatal(err)
} }
rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil) rh, err := storage.NewTestResourceHandler(resourceDir, &fakeBackend{}, nil, -1)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }