From 763fb7aa63266123c752f52ae1645cd9a2353624 Mon Sep 17 00:00:00 2001 From: lash Date: Tue, 23 Jan 2018 23:44:27 +0100 Subject: [PATCH 1/6] swarm/storage, swarm/api: Add error granularity for mut.rsrc --- swarm/api/api.go | 6 +-- swarm/api/http/server.go | 31 +++++++++++++-- swarm/storage/error.go | 13 ++++++ swarm/storage/resource.go | 83 ++++++++++++++++++++++++--------------- 4 files changed, 94 insertions(+), 39 deletions(-) create mode 100644 swarm/storage/error.go diff --git a/swarm/api/api.go b/swarm/api/api.go index 572b150da2..3a3e4a81d6 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -370,11 +370,7 @@ func (self *Api) ResourceLookup(ctx context.Context, name string, period uint32, var err error if version != 0 { if period == 0 { - currentblocknumber, err := self.resource.GetBlock(ctx) - if err != nil { - return nil, nil, fmt.Errorf("Could not determine latest block: %v", err) - } - period = self.resource.BlockToPeriod(name, currentblocknumber) + return nil, nil, storage.NewResourceError(storage.ErrInval, "Period can't be 0") } _, err = self.resource.LookupVersionByName(ctx, name, period, version, true) } else if period != 0 { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index a71edfdd23..3ad8503314 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -301,7 +301,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { } key, err := s.api.ResourceCreate(r.Context(), r.uri.Addr, frequency) if err != nil { - s.Error(w, r, fmt.Errorf("Resource creation failed: %v", err)) + s.translateResourceError(w, r, "Resource creation fail", err) return } outdata = key.Hex() @@ -314,7 +314,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { } _, _, _, err = s.api.ResourceUpdate(r.Context(), r.uri.Addr, data) if err != nil { - s.Error(w, r, fmt.Errorf("Update resource failed: %v", err)) + s.translateResourceError(w, r, "Mutable resource update fail", err) return } @@ -327,6 +327,31 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { w.WriteHeader(http.StatusOK) } +func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) { + code := 0 + defaulterr := fmt.Errorf("%s: %v", supErr, err) + rsrcerr, ok := err.(*storage.ResourceError) + if !ok { + code = rsrcerr.Code() + } + switch code { + case storage.ErrInval: + s.BadRequest(w, r, defaulterr.Error()) + case storage.ErrNoent, storage.ErrSync, storage.ErrNodata: + s.NotFound(w, r, defaulterr) + return + case storage.ErrAcces, storage.ErrNokey: + ShowError(w, &r.Request, defaulterr.Error(), http.StatusUnauthorized) + return + case storage.ErrFbig: + ShowError(w, &r.Request, defaulterr.Error(), http.StatusRequestEntityTooLarge) + return + } + + s.Error(w, r, defaulterr) + return +} + // Retrieve mutable resource updates: // bzz-resource:// - get latest update // bzz-resource:/// - get latest update on period n @@ -372,7 +397,7 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin return } if err != nil { - s.Error(w, r, fmt.Errorf("Mutable resource lookup failed: %v", err)) + s.translateResourceError(w, r, "Mutable resource lookup fail", err) return } log.Debug("Found update", "key", updateKey) diff --git a/swarm/storage/error.go b/swarm/storage/error.go new file mode 100644 index 0000000000..0ad79ded96 --- /dev/null +++ b/swarm/storage/error.go @@ -0,0 +1,13 @@ +package storage + +const ( + ErrCustom = iota + ErrNoent + ErrIO + ErrAcces + ErrInval + ErrFbig + ErrNodata + ErrNokey + ErrSync +) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 03235fa65a..22e2ffe56e 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -27,6 +27,30 @@ const ( hasherCount = 8 ) +type ResourceError struct { + code int + err string +} + +func (e *ResourceError) Error() string { + return e.err +} + +func (e *ResourceError) Code() int { + return e.code +} + +func NewResourceError(code int, s string) error { + r := &ResourceError{ + err: s, + } + switch code { + case ErrNoent, ErrIO, ErrAcces, ErrFbig, ErrInval, ErrNodata, ErrNokey, ErrSync: + r.code = code + } + return r +} + type Signature [signatureLength]byte type SignFunc func(common.Hash) (Signature, error) @@ -149,7 +173,7 @@ func NewResourceHandler(datadir string, cloudStore CloudStore, ethClient ethApi, path := filepath.Join(datadir, DbDirName) dbStore, err := NewDbStore(datadir, hashfunc, singletonSwarmDbCapacity, 0) if err != nil { - return nil, err + return nil, NewResourceError(ErrIO, fmt.Sprintf("datastore failed to initialize: %v", err)) } localStore := &LocalStore{ memStore: NewMemStore(dbStore, singletonSwarmDbCapacity), @@ -204,7 +228,7 @@ func (self *ResourceHandler) HashSize() int { func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, nil, errors.New("Resource does not exist or is not synced") + return nil, nil, NewResourceError(ErrNoent, "Resource does not exist or is not synced") } return rsrc.lastKey, rsrc.data, nil } @@ -213,7 +237,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, errors.New("Resource does not exist or is not synced") + return 0, NewResourceError(ErrNoent, "Resource does not exist or is not synced") } return rsrc.lastPeriod, nil } @@ -221,7 +245,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, errors.New("Resource does not exist or is not synced") + return 0, NewResourceError(ErrNoent, "Resource does not exist or is not synced") } return rsrc.version, nil } @@ -240,11 +264,11 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ // frequency 0 is invalid if frequency == 0 { - return nil, errors.New("Frequency cannot be 0") + return nil, NewResourceError(ErrInval, "Frequency cannot be 0") } if !isSafeName(name) { - return nil, fmt.Errorf("Invalid name: '%s'", name) + return nil, NewResourceError(ErrInval, fmt.Sprintf("Invalid name: '%s'", name)) } nameHash := self.nameHash(name) @@ -252,22 +276,22 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ if self.validator != nil { signature, err := self.validator.sign(nameHash) if err != nil { - return nil, fmt.Errorf("Sign fail: %v", err) + return nil, NewResourceError(ErrNokey, fmt.Sprintf("Sign fail: %v", err)) } addr, err := getAddressFromDataSig(nameHash, signature) if err != nil { - return nil, fmt.Errorf("Retrieve address from signature fail: %v", err) + return nil, NewResourceError(ErrNokey, fmt.Sprintf("Retrieve address from signature fail: %v", err)) } ok, err := self.validator.checkAccess(name, addr) if err != nil { return nil, err } else if !ok { - return nil, fmt.Errorf("Not owner of '%s'", name) + return nil, NewResourceError(ErrAcces, fmt.Sprintf("Not owner of '%s'", name)) } } // get our blockheight at this time - currentblock, err := self.GetBlock(ctx) + currentblock, err := self.getBlock(ctx) if err != nil { return nil, err } @@ -360,7 +384,7 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H if err != nil { return nil, err } - currentblock, err := self.GetBlock(ctx) + currentblock, err := self.getBlock(ctx) if err != nil { return nil, err } @@ -372,7 +396,7 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { if period == 0 { - return nil, errors.New("period must be >0") + return nil, NewResourceError(ErrInval, "period must be >0") } // start from the last possible block period, and iterate previous ones until we find a match @@ -408,7 +432,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) period-- } - return nil, errors.New("no updates found") + return nil, NewResourceError(ErrNoent, "no updates found") } // load existing mutable resource into resource struct @@ -425,7 +449,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref rsrc = &resource{} // make sure our name is safe to use if !isSafeName(name) { - return nil, fmt.Errorf("Invalid name '%s'", name) + return nil, NewResourceError(ErrInval, fmt.Sprintf("Invalid name '%s'", name)) } rsrc.name = &name rsrc.nameHash = nameHash @@ -438,7 +462,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref // minimum sanity check for chunk data if len(chunk.SData) != indexSize { - return nil, fmt.Errorf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize) + return nil, NewResourceError(ErrNodata, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize)) } rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8]) rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:]) @@ -457,7 +481,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve metadata from chunk data and check that it matches this mutable resource signature, period, version, name, data, err := self.parseUpdate(chunk.SData) if *rsrc.name != name { - return nil, fmt.Errorf("Update belongs to '%s', but have '%s'", name, *rsrc.name) + return nil, NewResourceError(ErrNodata, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name)) } log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version) // only check signature if validator is present @@ -465,7 +489,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( digest := self.keyDataHash(chunk.Key, data) _, err = getAddressFromDataSig(digest, *signature) if err != nil { - return nil, fmt.Errorf("Invalid signature: %v", err) + return nil, NewResourceError(ErrAcces, fmt.Sprintf("Invalid signature: %v", err)) } } @@ -484,16 +508,13 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve update metadata from chunk data // mirrors newUpdateChunk() func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, uint32, string, []byte, error) { - var err error cursor := 0 headerlength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) cursor += 2 datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+datalength+4) > len(chunkdata) { - err = fmt.Errorf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata)) - return nil, 0, 0, "", nil, err + return nil, 0, 0, "", nil, NewResourceError(ErrNodata, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata))) } - var period uint32 var version uint32 var name string @@ -537,22 +558,22 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt // get the cached information rsrc := self.getResource(name) if rsrc == nil { - return nil, errors.New("Resource object not in index") + return nil, NewResourceError(ErrNoent, "Resource object not in index") } if !rsrc.isSynced() { - return nil, errors.New("Resource object not in sync") + return nil, NewResourceError(ErrSync, "Resource object not in sync") } // an update can be only one chunk long datalimit := self.chunkSize() - int64(signaturelength-len(name)-4-4-2-2) if int64(len(data)) > datalimit { - return nil, fmt.Errorf("Data overflow: %d / %d bytes", len(data), datalimit) + return nil, NewResourceError(ErrFbig, fmt.Sprintf("Data overflow: %d / %d bytes", len(data), datalimit)) } // get our blockheight at this time and the next block of the update period currentblock, err := self.GetBlock(ctx) if err != nil { - return nil, err + return nil, NewResourceError(ErrIO, fmt.Sprintf("Could not get block height: %v", err)) } nextperiod := getNextPeriod(rsrc.startBlock, currentblock, rsrc.frequency) @@ -573,22 +594,22 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt digest := self.keyDataHash(key, data) sig, err := self.validator.sign(digest) if err != nil { - return nil, err + return nil, NewResourceError(ErrNokey, fmt.Sprintf("Sign fail: %v", err)) } signature = &sig // get the address of the signer (which also checks that it's a valid signature) addr, err := getAddressFromDataSig(digest, *signature) if err != nil { - return nil, fmt.Errorf("Invalid data/signature: %v", err) + return nil, NewResourceError(ErrNokey, fmt.Sprintf("Invalid data/signature: %v", err)) } // check if the signer has access to update ok, err := self.validator.checkAccess(name, addr) if err != nil { - return nil, err + return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err)) } else if !ok { - return nil, fmt.Errorf("Address %x does not have access to update %s", addr, name) + return nil, NewResourceError(ErrAcces, fmt.Sprintf("Address %x does not have access to update %s", addr, name)) } } @@ -600,7 +621,7 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt select { case <-chunk.dbStored: case <-timeout.C: - + return nil, NewResourceError(ErrIO, "chunk store timeout") } log.Trace("resource update", "name", name, "key", key, "currentblock", currentblock, "lastperiod", nextperiod, "version", version, "data", chunk.SData) @@ -618,7 +639,7 @@ func (self *ResourceHandler) Close() { self.ChunkStore.Close() } -func (self *ResourceHandler) GetBlock(ctx context.Context) (uint64, error) { +func (self *ResourceHandler) getBlock(ctx context.Context) (uint64, error) { blockheader, err := self.ethClient.HeaderByNumber(ctx, nil) if err != nil { return 0, err From 737fdb328d2d4185f0b02446cb26933744927f34 Mon Sep 17 00:00:00 2001 From: lash Date: Wed, 24 Jan 2018 02:32:53 +0100 Subject: [PATCH 2/6] swarm/api: Serve Resource Updates from bzz: scheme --- swarm/api/api.go | 24 ++++++++++ swarm/api/http/server.go | 85 +++++++++++++++++++++++------------ swarm/api/http/server_test.go | 72 ++++++++++++++++++++++++++--- swarm/api/manifest.go | 16 +++++++ swarm/storage/resource.go | 2 +- 5 files changed, 164 insertions(+), 35 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 3a3e4a81d6..8f73cfbdee 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -37,6 +37,18 @@ import ( var hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") +type ErrResourceReturn struct { + key string +} + +func (e *ErrResourceReturn) Error() string { + return "resourceupdate" +} + +func (e *ErrResourceReturn) Key() string { + return e.key +} + type Resolver interface { Resolve(string) (common.Hash, error) } @@ -145,6 +157,18 @@ func (self *Api) Get(key storage.Key, path string) (reader storage.LazySectionRe entry, _ := trie.getEntry(path) if entry != nil { + // we want to be able to serve Mutable Resource Updates transparently using the bzz:// scheme + // + // we use a special manifest hack for this purpose, which is pathless and where the resource root key + // is set as the hash of the manifest (see swarm/api/manifest.go:NewResourceManifest) + // + // to avoid taking a performance hit hacking a storage.LazySectionReader to wrap the resource key, + // we return a typed error instead. Since for all other purposes this is an invalid manifest, + // any normal interfacing code will just see an error fail accordingly. + if entry.ContentType == ResourceContentType { + log.Warn("resource type", "hash", entry.Hash) + return nil, entry.ContentType, http.StatusOK, &ErrResourceReturn{entry.Hash} + } key = common.Hex2Bytes(entry.Hash) status = entry.Status if status == http.StatusMultipleChoices { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 3ad8503314..335d6e15e8 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -43,6 +43,12 @@ import ( "github.com/rs/cors" ) +type resourceResponse struct { + Manifest storage.Key `json:"manifest"` + Resource string `json:"resource"` + Update storage.Key `json:"update"` +} + // ServerConfig is the basic configuration needed for the HTTP server and also // includes CORS settings. type ServerConfig struct { @@ -292,7 +298,7 @@ func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) { } func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { - var outdata string + var outdata []byte if r.uri.Path != "" { frequency, err := strconv.ParseUint(r.uri.Path, 10, 64) if err != nil { @@ -304,7 +310,21 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { s.translateResourceError(w, r, "Resource creation fail", err) return } - outdata = key.Hex() + m, err := s.api.NewResourceManifest(r.uri.Addr) + if err != nil { + s.Error(w, r, fmt.Errorf("Failed to create resource manifest: %v", err)) + return + } + rsrcResponse := &resourceResponse{ + Manifest: m, + Resource: r.uri.Addr, + Update: key, + } + outdata, err = json.Marshal(rsrcResponse) + if err != nil { + s.Error(w, r, fmt.Errorf("Failed to create json response for %v: error was: %v", r, err)) + return + } } data, err := ioutil.ReadAll(r.Body) @@ -318,40 +338,15 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) { return } - if outdata != "" { + if len(outdata) > 0 { w.Header().Add("Content-type", "text/plain") w.WriteHeader(http.StatusOK) - fmt.Fprint(w, outdata) + fmt.Fprint(w, string(outdata)) return } w.WriteHeader(http.StatusOK) } -func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) { - code := 0 - defaulterr := fmt.Errorf("%s: %v", supErr, err) - rsrcerr, ok := err.(*storage.ResourceError) - if !ok { - code = rsrcerr.Code() - } - switch code { - case storage.ErrInval: - s.BadRequest(w, r, defaulterr.Error()) - case storage.ErrNoent, storage.ErrSync, storage.ErrNodata: - s.NotFound(w, r, defaulterr) - return - case storage.ErrAcces, storage.ErrNokey: - ShowError(w, &r.Request, defaulterr.Error(), http.StatusUnauthorized) - return - case storage.ErrFbig: - ShowError(w, &r.Request, defaulterr.Error(), http.StatusRequestEntityTooLarge) - return - } - - s.Error(w, r, defaulterr) - return -} - // Retrieve mutable resource updates: // bzz-resource:// - get latest update // bzz-resource:/// - get latest update on period n @@ -405,6 +400,31 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data)) } +func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) { + code := 0 + defaulterr := fmt.Errorf("%s: %v", supErr, err) + rsrcerr, ok := err.(*storage.ResourceError) + if !ok { + code = rsrcerr.Code() + } + switch code { + case storage.ErrInval: + s.BadRequest(w, r, defaulterr.Error()) + case storage.ErrNoent, storage.ErrSync, storage.ErrNodata: + s.NotFound(w, r, defaulterr) + return + case storage.ErrAcces, storage.ErrNokey: + ShowError(w, &r.Request, defaulterr.Error(), http.StatusUnauthorized) + return + case storage.ErrFbig: + ShowError(w, &r.Request, defaulterr.Error(), http.StatusRequestEntityTooLarge) + return + } + + s.Error(w, r, defaulterr) + return +} + // HandleGet handles a GET request to // - bzz-raw:// and responds with the raw content stored at the // given storage key @@ -665,7 +685,14 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) { } reader, contentType, status, err := s.api.Get(key, r.uri.Path) + if err != nil { + // cheeky, cheeky hack. See swarm/api/api.go:Api.Get() for an explanation + if rsrcErr, ok := err.(*api.ErrResourceReturn); ok { + log.Trace("getting resource proxy", "err", rsrcErr.Key()) + s.handleGetResource(w, r, rsrcErr.Key()) + return + } switch status { case http.StatusNotFound: s.NotFound(w, r, err) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 9a80405ffc..3e695fc1d4 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -19,28 +19,45 @@ package http_test import ( "bytes" "crypto/rand" + "encoding/json" "errors" + "flag" "fmt" "io/ioutil" "net/http" + "os" "strings" "sync" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/swarm/api" swarm "github.com/ethereum/go-ethereum/swarm/api/client" "github.com/ethereum/go-ethereum/swarm/storage" "github.com/ethereum/go-ethereum/swarm/testutil" ) +func init() { + verbose := flag.Bool("v", false, "verbose") + flag.Parse() + if *verbose { + log.Root().SetHandler(log.CallerFileHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))) + } +} + +type resourceResponse struct { + Manifest storage.Key `json:"manifest"` + Resource string `json:"resource"` + Update storage.Key `json:"update"` +} + func TestBzzResource(t *testing.T) { srv := testutil.NewTestSwarmServer(t) defer srv.Close() // our mutable resource "name" - keybytes := make([]byte, common.HashLength) - copy(keybytes, []byte{42}) + keybytes := []byte("foo") srv.Hasher.Reset() srv.Hasher.Write([]byte(fmt.Sprintf("%x", keybytes))) keybyteshash := fmt.Sprintf("%x", srv.Hasher.Sum(nil)) @@ -66,10 +83,55 @@ func TestBzzResource(t *testing.T) { if err != nil { t.Fatal(err) } - if !bytes.Equal(b, []byte(keybyteshash)) { - t.Fatalf("resource update hash mismatch, expected '%s' got '%s'", keybyteshash, b) + rsrcResp := &resourceResponse{} + err = json.Unmarshal(b, rsrcResp) + if err != nil { + t.Fatalf("data %s could not be unmarshaled: %v", b, err) + } + if rsrcResp.Update.Hex() != keybyteshash { + t.Fatalf("Response resource key mismatch, expected '%s', got '%s'", keybyteshash, rsrcResp.Resource) + } + + // get manifest + url = fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, rsrcResp.Manifest) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + manifest := &api.Manifest{} + err = json.Unmarshal(b, manifest) + if err != nil { + t.Fatal(err) + } + if len(manifest.Entries) != 1 { + t.Fatalf("Manifest has %d entries", len(manifest.Entries)) + } + if manifest.Entries[0].Hash != rsrcResp.Resource { + t.Fatalf("Expected manifest path '%s', got '%s'", keybyteshash, manifest.Entries[0].Hash) + } + + // get bzz manifest transparent resource resolve + url = fmt.Sprintf("%s/bzz:/%s", srv.URL, rsrcResp.Manifest) + resp, err = http.Get(url) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("err %s", resp.Status) + } + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) } - t.Logf("creatreturn %v / %v", keybyteshash, b) // get latest update (1.1) through resource directly url = fmt.Sprintf("%s/bzz-resource:/%x", srv.URL, keybytes) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index fde086b7ac..660e5131fa 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -69,6 +69,22 @@ func (a *Api) NewManifest() (storage.Key, error) { return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{}) } +// Manifest hack for supporting Mutable Resource Updates from the bzz: scheme +// see swarm/api/api.go:Api.Get() for more information +func (a *Api) NewResourceManifest(resourceKey string) (storage.Key, error) { + var manifest Manifest + entry := ManifestEntry{ + Hash: resourceKey, + ContentType: ResourceContentType, + } + manifest.Entries = append(manifest.Entries, entry) + data, err := json.Marshal(&manifest) + if err != nil { + return nil, err + } + return a.Store(bytes.NewReader(data), int64(len(data)), &sync.WaitGroup{}) +} + // ManifestWriter is used to add and remove entries from an underlying manifest type ManifestWriter struct { api *Api diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 22e2ffe56e..3606533e09 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -571,7 +571,7 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt } // get our blockheight at this time and the next block of the update period - currentblock, err := self.GetBlock(ctx) + currentblock, err := self.getBlock(ctx) if err != nil { return nil, NewResourceError(ErrIO, fmt.Sprintf("Could not get block height: %v", err)) } From 5f2378dfc51deb7122d4e8b9b53dd5044389982c Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 26 Jan 2018 05:28:32 +0100 Subject: [PATCH 3/6] swarm/api, swarm/storage: Change to full error names --- swarm/api/api.go | 2 +- swarm/api/http/server.go | 24 ++++++++++---------- swarm/storage/error.go | 16 ++++++------- swarm/storage/resource.go | 47 +++++++++++++++++++++------------------ 4 files changed, 46 insertions(+), 43 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 8f73cfbdee..53aa5392a0 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -394,7 +394,7 @@ func (self *Api) ResourceLookup(ctx context.Context, name string, period uint32, var err error if version != 0 { if period == 0 { - return nil, nil, storage.NewResourceError(storage.ErrInval, "Period can't be 0") + return nil, nil, storage.NewResourceError(storage.ErrInvalidValue, "Period can't be 0") } _, err = self.resource.LookupVersionByName(ctx, name, period, version, true) } else if period != 0 { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 335d6e15e8..0dcc4e0fca 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -402,26 +402,26 @@ func (s *Server) handleGetResource(w http.ResponseWriter, r *Request, name strin func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) { code := 0 - defaulterr := fmt.Errorf("%s: %v", supErr, err) - rsrcerr, ok := err.(*storage.ResourceError) + defaultErr := fmt.Errorf("%s: %v", supErr, err) + rsrcErr, ok := err.(*storage.ResourceError) if !ok { - code = rsrcerr.Code() + code = rsrcErr.Code() } switch code { - case storage.ErrInval: - s.BadRequest(w, r, defaulterr.Error()) - case storage.ErrNoent, storage.ErrSync, storage.ErrNodata: - s.NotFound(w, r, defaulterr) + case storage.ErrInvalidValue: + s.BadRequest(w, r, defaultErr.Error()) + case storage.ErrNotFound, storage.ErrNotSynced, storage.ErrNothingToReturn: + s.NotFound(w, r, defaultErr) return - case storage.ErrAcces, storage.ErrNokey: - ShowError(w, &r.Request, defaulterr.Error(), http.StatusUnauthorized) + case storage.ErrUnauthorized, storage.ErrInvalidSignature: + ShowError(w, &r.Request, defaultErr.Error(), http.StatusUnauthorized) return - case storage.ErrFbig: - ShowError(w, &r.Request, defaulterr.Error(), http.StatusRequestEntityTooLarge) + case storage.ErrDataOverflow: + ShowError(w, &r.Request, defaultErr.Error(), http.StatusRequestEntityTooLarge) return } - s.Error(w, r, defaulterr) + s.Error(w, r, defaultErr) return } diff --git a/swarm/storage/error.go b/swarm/storage/error.go index 0ad79ded96..56e459731c 100644 --- a/swarm/storage/error.go +++ b/swarm/storage/error.go @@ -1,13 +1,13 @@ package storage const ( - ErrCustom = iota - ErrNoent + ErrNotFound = iota ErrIO - ErrAcces - ErrInval - ErrFbig - ErrNodata - ErrNokey - ErrSync + ErrUnauthorized + ErrInvalidValue + ErrDataOverflow + ErrNothingToReturn + ErrInvalidSignature + ErrNotSynced + ErrCnt ) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 3606533e09..e99708bed7 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -41,11 +41,14 @@ func (e *ResourceError) Code() int { } func NewResourceError(code int, s string) error { + if code < 0 || code >= ErrCnt { + panic("no such error code!") + } r := &ResourceError{ err: s, } switch code { - case ErrNoent, ErrIO, ErrAcces, ErrFbig, ErrInval, ErrNodata, ErrNokey, ErrSync: + case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced: r.code = code } return r @@ -228,7 +231,7 @@ func (self *ResourceHandler) HashSize() int { func (self *ResourceHandler) GetContent(name string) (Key, []byte, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return nil, nil, NewResourceError(ErrNoent, "Resource does not exist or is not synced") + return nil, nil, NewResourceError(ErrNotFound, "Resource does not exist or is not synced") } return rsrc.lastKey, rsrc.data, nil } @@ -237,7 +240,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, NewResourceError(ErrNoent, "Resource does not exist or is not synced") + return 0, NewResourceError(ErrNotFound, "Resource does not exist or is not synced") } return rsrc.lastPeriod, nil } @@ -245,7 +248,7 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) if rsrc == nil || !rsrc.isSynced() { - return 0, NewResourceError(ErrNoent, "Resource does not exist or is not synced") + return 0, NewResourceError(ErrNotFound, "Resource does not exist or is not synced") } return rsrc.version, nil } @@ -264,11 +267,11 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ // frequency 0 is invalid if frequency == 0 { - return nil, NewResourceError(ErrInval, "Frequency cannot be 0") + return nil, NewResourceError(ErrInvalidValue, "Frequency cannot be 0") } if !isSafeName(name) { - return nil, NewResourceError(ErrInval, fmt.Sprintf("Invalid name: '%s'", name)) + return nil, NewResourceError(ErrInvalidValue, fmt.Sprintf("Invalid name: '%s'", name)) } nameHash := self.nameHash(name) @@ -276,17 +279,17 @@ func (self *ResourceHandler) NewResource(ctx context.Context, name string, frequ if self.validator != nil { signature, err := self.validator.sign(nameHash) if err != nil { - return nil, NewResourceError(ErrNokey, fmt.Sprintf("Sign fail: %v", err)) + return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err)) } addr, err := getAddressFromDataSig(nameHash, signature) if err != nil { - return nil, NewResourceError(ErrNokey, fmt.Sprintf("Retrieve address from signature fail: %v", err)) + return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Retrieve address from signature fail: %v", err)) } ok, err := self.validator.checkAccess(name, addr) if err != nil { return nil, err } else if !ok { - return nil, NewResourceError(ErrAcces, fmt.Sprintf("Not owner of '%s'", name)) + return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Not owner of '%s'", name)) } } @@ -396,7 +399,7 @@ func (self *ResourceHandler) LookupLatest(ctx context.Context, nameHash common.H func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint32, refresh bool) (*resource, error) { if period == 0 { - return nil, NewResourceError(ErrInval, "period must be >0") + return nil, NewResourceError(ErrInvalidValue, "period must be >0") } // start from the last possible block period, and iterate previous ones until we find a match @@ -432,7 +435,7 @@ func (self *ResourceHandler) lookup(rsrc *resource, period uint32, version uint3 log.Trace("rsrc update not found, checking previous period", "period", period, "key", key) period-- } - return nil, NewResourceError(ErrNoent, "no updates found") + return nil, NewResourceError(ErrNotFound, "no updates found") } // load existing mutable resource into resource struct @@ -449,7 +452,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref rsrc = &resource{} // make sure our name is safe to use if !isSafeName(name) { - return nil, NewResourceError(ErrInval, fmt.Sprintf("Invalid name '%s'", name)) + return nil, NewResourceError(ErrInvalidValue, fmt.Sprintf("Invalid name '%s'", name)) } rsrc.name = &name rsrc.nameHash = nameHash @@ -462,7 +465,7 @@ func (self *ResourceHandler) loadResource(nameHash common.Hash, name string, ref // minimum sanity check for chunk data if len(chunk.SData) != indexSize { - return nil, NewResourceError(ErrNodata, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize)) + return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Invalid chunk length %d, should be %d", len(chunk.SData), indexSize)) } rsrc.startBlock = binary.LittleEndian.Uint64(chunk.SData[:8]) rsrc.frequency = binary.LittleEndian.Uint64(chunk.SData[8:]) @@ -481,7 +484,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( // retrieve metadata from chunk data and check that it matches this mutable resource signature, period, version, name, data, err := self.parseUpdate(chunk.SData) if *rsrc.name != name { - return nil, NewResourceError(ErrNodata, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name)) + return nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Update belongs to '%s', but have '%s'", name, *rsrc.name)) } log.Trace("update", "name", *rsrc.name, "rootkey", rsrc.nameHash, "updatekey", chunk.Key, "period", period, "version", version) // only check signature if validator is present @@ -489,7 +492,7 @@ func (self *ResourceHandler) updateResourceIndex(rsrc *resource, chunk *Chunk) ( digest := self.keyDataHash(chunk.Key, data) _, err = getAddressFromDataSig(digest, *signature) if err != nil { - return nil, NewResourceError(ErrAcces, fmt.Sprintf("Invalid signature: %v", err)) + return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Invalid signature: %v", err)) } } @@ -513,7 +516,7 @@ func (self *ResourceHandler) parseUpdate(chunkdata []byte) (*Signature, uint32, cursor += 2 datalength := binary.LittleEndian.Uint16(chunkdata[cursor : cursor+2]) if int(headerlength+datalength+4) > len(chunkdata) { - return nil, 0, 0, "", nil, NewResourceError(ErrNodata, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata))) + return nil, 0, 0, "", nil, NewResourceError(ErrNothingToReturn, fmt.Sprintf("Reported headerlength %d + datalength %d longer than actual chunk data length %d", headerlength, datalength, len(chunkdata))) } var period uint32 var version uint32 @@ -558,16 +561,16 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt // get the cached information rsrc := self.getResource(name) if rsrc == nil { - return nil, NewResourceError(ErrNoent, "Resource object not in index") + return nil, NewResourceError(ErrNotFound, "Resource object not in index") } if !rsrc.isSynced() { - return nil, NewResourceError(ErrSync, "Resource object not in sync") + return nil, NewResourceError(ErrNotSynced, "Resource object not in sync") } // an update can be only one chunk long datalimit := self.chunkSize() - int64(signaturelength-len(name)-4-4-2-2) if int64(len(data)) > datalimit { - return nil, NewResourceError(ErrFbig, fmt.Sprintf("Data overflow: %d / %d bytes", len(data), datalimit)) + return nil, NewResourceError(ErrDataOverflow, fmt.Sprintf("Data overflow: %d / %d bytes", len(data), datalimit)) } // get our blockheight at this time and the next block of the update period @@ -594,14 +597,14 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt digest := self.keyDataHash(key, data) sig, err := self.validator.sign(digest) if err != nil { - return nil, NewResourceError(ErrNokey, fmt.Sprintf("Sign fail: %v", err)) + return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Sign fail: %v", err)) } signature = &sig // get the address of the signer (which also checks that it's a valid signature) addr, err := getAddressFromDataSig(digest, *signature) if err != nil { - return nil, NewResourceError(ErrNokey, fmt.Sprintf("Invalid data/signature: %v", err)) + return nil, NewResourceError(ErrInvalidSignature, fmt.Sprintf("Invalid data/signature: %v", err)) } // check if the signer has access to update @@ -609,7 +612,7 @@ func (self *ResourceHandler) Update(ctx context.Context, name string, data []byt if err != nil { return nil, NewResourceError(ErrIO, fmt.Sprintf("Access check fail: %v", err)) } else if !ok { - return nil, NewResourceError(ErrAcces, fmt.Sprintf("Address %x does not have access to update %s", addr, name)) + return nil, NewResourceError(ErrUnauthorized, fmt.Sprintf("Address %x does not have access to update %s", addr, name)) } } From 6e708b758ab9a7ae261033dbfb3a9d86e4432e45 Mon Sep 17 00:00:00 2001 From: lash Date: Fri, 26 Jan 2018 07:35:00 +0100 Subject: [PATCH 4/6] swarm/api, swarm/network: Delint + nil chan err in protocoltester --- swarm/api/http/server.go | 1 - swarm/network/protocol_test.go | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 0dcc4e0fca..d4c4728079 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -422,7 +422,6 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr } s.Error(w, r, defaultErr) - return } // HandleGet handles a GET request to diff --git a/swarm/network/protocol_test.go b/swarm/network/protocol_test.go index e8ec4ebc37..2cc55e02ce 100644 --- a/swarm/network/protocol_test.go +++ b/swarm/network/protocol_test.go @@ -82,7 +82,11 @@ func newBzzBaseTester(t *testing.T, n int, addr *BzzAddr, spec *protocols.Spec, cs := make(map[string]chan bool) srv := func(p *bzzPeer) error { - defer close(cs[p.ID().String()]) + defer func() { + if cs[p.ID().String()] != nil { + close(cs[p.ID().String()]) + } + }() return run(p) } From c7a45a512ca897a0b35bdc8d5d0cfa47f90df3f8 Mon Sep 17 00:00:00 2001 From: lash Date: Thu, 1 Feb 2018 15:44:16 +0100 Subject: [PATCH 5/6] swarm/storage: Add explicit error on unsynced resource --- swarm/storage/resource.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index e99708bed7..8884be09b3 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -247,8 +247,10 @@ func (self *ResourceHandler) GetLastPeriod(name string) (uint32, error) { func (self *ResourceHandler) GetVersion(name string) (uint32, error) { rsrc := self.getResource(name) - if rsrc == nil || !rsrc.isSynced() { - return 0, NewResourceError(ErrNotFound, "Resource does not exist or is not synced") + if rsrc == nil { + return 0, NewResourceError(ErrNotFound, "Resource does not exist") + } else if !rsrc.isSynced() { + return 0, NewResourceError(ErrNotSynced, "Resource is not synced") } return rsrc.version, nil } From 4a9076d0e4bbfe7108f008496294e55c14306b5f Mon Sep 17 00:00:00 2001 From: lash Date: Mon, 5 Feb 2018 19:28:40 +0100 Subject: [PATCH 6/6] swarm/storage: Remove redundant error code switch --- swarm/storage/resource.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/swarm/storage/resource.go b/swarm/storage/resource.go index 8884be09b3..5afef992f3 100644 --- a/swarm/storage/resource.go +++ b/swarm/storage/resource.go @@ -45,11 +45,8 @@ func NewResourceError(code int, s string) error { panic("no such error code!") } r := &ResourceError{ - err: s, - } - switch code { - case ErrNotFound, ErrIO, ErrUnauthorized, ErrInvalidValue, ErrDataOverflow, ErrNothingToReturn, ErrInvalidSignature, ErrNotSynced: - r.code = code + err: s, + code: code, } return r }