From fb3b4fa49c5c79def7d0f8f93577ab4f1016598d Mon Sep 17 00:00:00 2001 From: "Daniel A. Nagy" Date: Thu, 4 Feb 2016 12:12:39 +0100 Subject: [PATCH] support for separate url schemes for dns enabled, immutable and raw manifest * Changed protocol choice. Missing proper distinction between mutable and immutable Swarm protocols. * Modified API to allow for immutable storage. * Protocol handler installation updated. * Roundtripper changed to reflect http API changes * Change demo/example code to work with new protocol spec. * More meaningful logging, including request method. * Proxy all HTTP methods correctly. * Upload script modified for the new protocol specification. * Uploading shell script (example for raw http API) updated, tested. * Fixing crash while deleting non-existing manifest entries. HT to zsfelfoldi * No leading slash in typical manifests. * Deleting images works * Adding images works. * Change tests to conform with new API. Semantics of tests not changed. * Additional error handling in roundtripper. * fix roundtripper test --- swarm/api/api.go | 37 ++++++++++++++------------ swarm/api/api_test.go | 2 +- swarm/api/filesystem.go | 2 +- swarm/api/filesystem_test.go | 10 +++---- swarm/api/http/roundtripper.go | 12 ++++++--- swarm/api/http/roundtripper_test.go | 4 +-- swarm/api/http/server.go | 41 ++++++++++++++++++++--------- swarm/api/manifest.go | 5 +++- swarm/api/storage.go | 4 +-- swarm/cmd/bzzup.sh | 8 +++--- swarm/examples/album/index.js | 12 ++++----- swarm/examples/bzzhandler.html | 13 ++++++++- 12 files changed, 94 insertions(+), 56 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index ecd18a12a5..12cb8673bf 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -52,11 +52,15 @@ func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key stor } // DNS Resolver -func (self *Api) Resolve(hostPort string) (contentHash storage.Key, err error) { +func (self *Api) Resolve(hostPort string, nameresolver bool) (contentHash storage.Key, err error) { if hashMatcher.MatchString(hostPort) || self.dns == nil { - glog.V(logger.Debug).Infof("[BZZ] host is a contentHash: '%v'", contentHash) + glog.V(logger.Debug).Infof("[BZZ] host is a contentHash: '%v'", hostPort) return storage.Key(common.Hex2Bytes(hostPort)), nil } + if !nameresolver { + err = fmt.Errorf("'%s' is not a content hash value.", hostPort) + return + } contentHash, err = self.dns.Resolve(hostPort) if err != nil { err = ErrResolve(err) @@ -73,28 +77,27 @@ func parse(uri string) (hostPort, path string) { return } // beginning with slash is now optional - if len(parts[0]) == 0 { + for len(parts[i]) == 0 { i++ } hostPort = parts[i] - if len(parts) > i+1 { - path = parts[i+1] - if len(parts) == 3 { - path += "/" + parts[2] + for i < len(parts)-1 { + i++ + if len(path) > 0 { + path = path + "/" + parts[i] + } else { + path = parts[i] } - path += "/" - } - if len(path) > 0 { - path = "/" + path } glog.V(logger.Debug).Infof("[BZZ] Swarm: host: '%s', path '%s' requested.", hostPort, path) return } -func (self *Api) parseAndResolve(uri string) (contentHash storage.Key, hostPort, path string, err error) { +func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash storage.Key, hostPort, path string, err error) { hostPort, path = parse(uri) //resolving host and port - contentHash, err = self.Resolve(hostPort) + contentHash, err = self.Resolve(hostPort, nameresolver) + glog.V(logger.Debug).Infof("[BZZ] Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path) return } @@ -119,9 +122,9 @@ func (self *Api) Put(content, contentType string) (string, error) { // Get uses iterative manifest retrieval and prefix matching // to resolve path to content using dpa retrieve // it returns a section reader, mimeType, status and an error -func (self *Api) Get(uri string) (reader storage.SectionReader, mimeType string, status int, err error) { +func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) { - key, _, path, err := self.parseAndResolve(uri) + key, _, path, err := self.parseAndResolve(uri, nameresolver) trie, err := loadManifest(self.dpa, key) if err != nil { @@ -144,8 +147,8 @@ func (self *Api) Get(uri string) (reader storage.SectionReader, mimeType string, return } -func (self *Api) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { - root := common.Hex2Bytes(rootHash) +func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) { + root, _, path, err := self.parseAndResolve(uri, nameresolver) trie, err := loadManifest(self.dpa, root) if err != nil { return diff --git a/swarm/api/api_test.go b/swarm/api/api_test.go index 7db209c9d4..0ee3ca8df7 100644 --- a/swarm/api/api_test.go +++ b/swarm/api/api_test.go @@ -63,7 +63,7 @@ func expResponse(content string, mimeType string, status int) *Response { // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { func testGet(t *testing.T, api *Api, bzzhash string) *testResponse { - reader, mimeType, status, err := api.Get(bzzhash) + reader, mimeType, status, err := api.Get(bzzhash, true) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/swarm/api/filesystem.go b/swarm/api/filesystem.go index eddd72dd0b..3bd4a318d3 100644 --- a/swarm/api/filesystem.go +++ b/swarm/api/filesystem.go @@ -167,7 +167,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error { } //resolving host and port - key, _, path, err := self.api.parseAndResolve(bzzpath) + key, _, path, err := self.api.parseAndResolve(bzzpath, true) if err != nil { return err } diff --git a/swarm/api/filesystem_test.go b/swarm/api/filesystem_test.go index 174d8833a1..e0bb8915a9 100644 --- a/swarm/api/filesystem_test.go +++ b/swarm/api/filesystem_test.go @@ -58,7 +58,7 @@ func TestApiDirUpload0(t *testing.T) { resp = testGet(t, api, bzzhash+"/img/logo.png") exp = expResponse(content, "image/png", 0) - _, _, _, err = api.Get(bzzhash) + _, _, _, err = api.Get(bzzhash, true) if err == nil { t.Fatalf("expected error: %v", err) } @@ -91,17 +91,17 @@ func TestApiDirUploadModify(t *testing.T) { return } - bzzhash, err = api.Modify(bzzhash, "index.html", "", "") + bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true) if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash, "index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8") + bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) if err != nil { t.Errorf("unexpected error: %v", err) return } - bzzhash, err = api.Modify(bzzhash, "img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8") + bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true) if err != nil { t.Errorf("unexpected error: %v", err) return @@ -120,7 +120,7 @@ func TestApiDirUploadModify(t *testing.T) { resp = testGet(t, api, bzzhash+"/index.css") exp = expResponse(content, "text/css", 0) - _, _, _, err = api.Get(bzzhash) + _, _, _, err = api.Get(bzzhash, true) if err == nil { t.Errorf("expected error: %v", err) } diff --git a/swarm/api/http/roundtripper.go b/swarm/api/http/roundtripper.go index da6a63a255..69c501e5be 100644 --- a/swarm/api/http/roundtripper.go +++ b/swarm/api/http/roundtripper.go @@ -20,8 +20,8 @@ import ( client := httpclient.New() // for (private) swarm proxy running locally client.RegisterScheme("bzz", &http.RoundTripper{Port: port}) -// for public swarm gateway -client.RegisterScheme(scheme, &http.RoundTripper{Host: host, Port: port}) +client.RegisterScheme("bzzi", &http.RoundTripper{Port: port}) +client.RegisterScheme("bzzr", &http.RoundTripper{Port: port}) The port you give the Roundtripper is the port the swarm proxy is listening on. If Host is left empty, localhost is assumed. @@ -43,7 +43,11 @@ func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err if len(host) == 0 { host = "localhost" } - url := fmt.Sprintf("http://%s:%s/%s/%s", host, self.Port, req.URL.Host, req.URL.Path) + url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path) glog.V(logger.Info).Infof("[BZZ] roundtripper: proxying request '%s' to '%s'", req.RequestURI, url) - return http.Get(url) + reqProxy, err := http.NewRequest(req.Method, url, req.Body) + if err != nil { + return nil, err + } + return http.DefaultClient.Do(reqProxy) } diff --git a/swarm/api/http/roundtripper_test.go b/swarm/api/http/roundtripper_test.go index 51bcd2e41c..1bed107887 100644 --- a/swarm/api/http/roundtripper_test.go +++ b/swarm/api/http/roundtripper_test.go @@ -43,8 +43,8 @@ func TestRoundTripper(t *testing.T) { t.Errorf("expected no error, got %v", err) return } - if string(content) != "/test.com/path" { - t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/test.com/path", string(content)) + if string(content) != "/HTTP/1.1:/test.com/path" { + t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content)) } } diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index efa3651652..6ac2b113e6 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -22,8 +22,8 @@ const ( ) var ( - bzzPrefix = regexp.MustCompile("^/+bzz:/+") - rawUrl = regexp.MustCompile("^/+raw/*") + // accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw) + bzzPrefix = regexp.MustCompile("^/+bzz[ir]?:/+") trailingSlashes = regexp.MustCompile("/+$") // forever = func() time.Time { return time.Unix(0, 0) } forever = time.Now @@ -62,20 +62,37 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { // return // } // } - glog.V(logger.Debug).Infof("[BZZ] Swarm: HTTP request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")) + glog.V(logger.Debug).Infof("[BZZ] Swarm: HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept")) uri := requestURL.Path - var raw bool + var raw, nameresolver bool + var proto string // HTTP-based URL protocol handler - uri = bzzPrefix.ReplaceAllString(uri, "") glog.V(logger.Debug).Infof("[BZZ] Swarm: BZZ request URI: '%s'", uri) - path := rawUrl.ReplaceAllStringFunc(uri, func(string) string { - raw = true + path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string { + proto = p return "" }) - glog.V(logger.Debug).Infof("[BZZ] Swarm: %s request '%s' received.", r.Method, uri) + // protocol identification (ugly) + if proto == "" { + if glog.V(logger.Error) { + glog.Errorf( + "[BZZ] Swarm: Protocol error in request `%s`.", + uri, + ) + http.Error(w, "BZZ protocol error", http.StatusBadRequest) + return + } + } + raw = proto[1:5] == "bzzr" + nameresolver = proto[1:5] != "bzzi" + + glog.V(logger.Debug).Infof( + "[BZZ] Swarm: %s request over protocol %s '%s' received.", + r.Method, proto, path, + ) switch { case r.Method == "POST" || r.Method == "PUT": @@ -107,7 +124,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { mime := r.Header.Get("Content-Type") // TODO proper root hash separation glog.V(logger.Debug).Infof("[BZZ] Modify '%s' to store %v as '%s'.", path, key.Log(), mime) - newKey, err := a.Modify(path[:64], path[65:], common.Bytes2Hex(key), mime) + newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver) if err == nil { glog.V(logger.Debug).Infof("[BZZ] Swarm replaced manifest by '%s'", newKey) w.Header().Set("Content-Type", "text/plain") @@ -125,7 +142,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } else { path = api.RegularSlashes(path) glog.V(logger.Debug).Infof("[BZZ] Delete '%s'.", path) - newKey, err := a.Modify(path[:64], path[65:], "", "") + newKey, err := a.Modify(path, "", "", nameresolver) if err == nil { glog.V(logger.Debug).Infof("[BZZ] Swarm replaced manifest by '%s'", newKey) w.Header().Set("Content-Type", "text/plain") @@ -139,7 +156,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { path = trailingSlashes.ReplaceAllString(path, "") if raw { // resolving host - key, err := a.Resolve(path) + key, err := a.Resolve(path, nameresolver) if err != nil { glog.V(logger.Error).Infof("[BZZ] Swarm: %v", err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -166,7 +183,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { glog.V(logger.Debug).Infof("[BZZ] Swarm: Structured GET request '%s' received.", uri) - reader, mimeType, status, err := a.Get(path) + reader, mimeType, status, err := a.Get(path, nameresolver) if err != nil { if _, ok := err.(api.ErrResolve); ok { glog.V(logger.Debug).Infof("[BZZ] Swarm: %v", err) diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 42942a0213..0337382ff0 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -145,7 +145,10 @@ func (self *manifestTrie) deleteEntry(path string) { b := byte(path[0]) entry := self.entries[b] - if (entry != nil) && (entry.Path == path) { + if entry == nil { + return + } + if entry.Path == path { self.entries[b] = nil return } diff --git a/swarm/api/storage.go b/swarm/api/storage.go index 6563c75ae0..82b8f12fba 100644 --- a/swarm/api/storage.go +++ b/swarm/api/storage.go @@ -37,7 +37,7 @@ func (self *Storage) Put(content, contentType string) (string, error) { // the actual size of which is given in len(resp.Content), while the expected // size is resp.Size func (self *Storage) Get(bzzpath string) (*Response, error) { - reader, mimeType, status, err := self.api.Get(bzzpath) + reader, mimeType, status, err := self.api.Get(bzzpath, true) if err != nil { return nil, err } @@ -52,5 +52,5 @@ func (self *Storage) Get(bzzpath string) (*Response, error) { } func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) { - return self.api.Modify(rootHash, path, contentHash, contentType) + return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true) } diff --git a/swarm/cmd/bzzup.sh b/swarm/cmd/bzzup.sh index dc6490ad70..1f8988a0dc 100755 --- a/swarm/cmd/bzzup.sh +++ b/swarm/cmd/bzzup.sh @@ -9,9 +9,9 @@ if [[ ! -z "$2" ]]; then fi if [ -f "$1" ]; then -hash=`wget -q -O- --post-file="$1" http://localhost:$port/raw` +hash=`wget -q -O- --post-file="$1" "http://localhost:$port/bzzr:/"` mime=`mimetype -b "$1"` -wget -q -O- --post-data="$delimiter\"hash\":\"$hash\",\"contentType\":\"$mime\"}]}" http://localhost:8500/raw +wget -q -O- --post-data="$delimiter\"hash\":\"$hash\",\"contentType\":\"$mime\"}]}" "http://localhost:$port/bzzr:/" echo else @@ -28,7 +28,7 @@ do name=`echo "$path" | cut -c3-` [ _`basename "$name"` = "_$INDEX" ] && name=`dirname "$name"` echo -n "$delimiter" -hash=`wget -q -O- --post-file="$path" http://localhost:$port/raw` +hash=`wget -q -O- --post-file="$path" "http://localhost:$port/bzzr:/"` mime=`mimetype -b "$path"` if [ "_$name" = '_.' ]; then echo -n "\"hash\":\"$hash\",\"contentType\":\"$mime\"" @@ -38,7 +38,7 @@ fi delimiter='},{' done -echo -n '}]}') | wget -q -O- --post-data=`cat` http://localhost:$port/raw +echo -n '}]}') | wget -q -O- --post-data=`cat` "http://localhost:$port/bzzr:/" echo popd > /dev/null diff --git a/swarm/examples/album/index.js b/swarm/examples/album/index.js index 5bce32b3e3..aa7b46054d 100644 --- a/swarm/examples/album/index.js +++ b/swarm/examples/album/index.js @@ -288,7 +288,7 @@ function uploadFile(files, nr, uri) { var xhr = new XMLHttpRequest(); xhr.onreadystatechange = function() { if (xhr.readyState === 4) { var i = xhr.responseText; - window.location.replace("/" + i + "/"); + window.location.replace("/bzz:/" + i + "/"); }}; sendImgs(xhr, uri); } @@ -327,9 +327,9 @@ function uploadFile(files, nr, uri) { imgData[0] = "imgs/" + file.name; imgData[1] = [img.naturalWidth, img.naturalHeight]; imgs.data.splice(eidx, 0, {img: imgData, thumb: thumbData, blur: blur}); - uploadFile(files, nr + 1, "/" + i + "/"); + uploadFile(files, nr + 1, "/bzz:/" + i + "/"); } - img.src = "/" + i + "/imgs/" + file.name; + img.src = "/bzz:/" + i + "/imgs/" + file.name; return; }}; xhr.open("PUT", uri + "imgs/" + file.name, true); @@ -361,9 +361,9 @@ function deleteImg() var xhrd = new XMLHttpRequest(); xhrd.onreadystatechange = function () { if (xhrd.readyState === 4) { var j = xhrd.responseText; - window.location.replace("/" + j + "/"); + window.location.replace("/bzz:/" + j + "/"); }}; - xhrd.open("DELETE", "/" + i + "/" + fname, true); + xhrd.open("DELETE", "/bzz%3A/" + i + "/" + fname, true); xhrd.send(); }}; @@ -378,7 +378,7 @@ function moveUpDown(off) var xhr = new XMLHttpRequest(); xhr.onreadystatechange = function () { if (xhr.readyState === 4) { var i = xhr.responseText; - window.location.replace("/" + i + "/#" + (eidx + off)); + window.location.replace("/bzz:/" + i + "/#" + (eidx + off)); }}; sendImgs(xhr, ""); } diff --git a/swarm/examples/bzzhandler.html b/swarm/examples/bzzhandler.html index 0d7a8b01a4..cbd2403f95 100644 --- a/swarm/examples/bzzhandler.html +++ b/swarm/examples/bzzhandler.html @@ -6,10 +6,21 @@ navigator.registerProtocolHandler("bzz", "http://localhost:8500/%s", "Swarm handler"); + navigator.registerProtocolHandler("bzzi", + "http://localhost:8500/%s", + "Swarm immutable handler"); + navigator.registerProtocolHandler("bzzr", + "http://localhost:8500/%s", + "Swarm raw handler");

Register Swarm protocol handler

-

This web page will install a web protocol handler for the bzz: protocol.

+

This web page will install a web protocol handler for + bzz:, + bzzi: and + bzzr: + protocols. +