support for separate url schemes for dns enabled, immutable and raw manifest

* Changed protocol choice. Missing proper distinction between mutable and immutable Swarm protocols.

  * Modified API to allow for immutable storage.

  * Protocol handler installation updated.

  * Roundtripper changed to reflect http API changes

  * Change demo/example code to work with new protocol spec.

  * More meaningful logging, including request method.

  * Proxy all HTTP methods correctly.

  * Upload script modified for the new protocol specification.

  * Uploading shell script (example for raw http API) updated, tested.

  * Fixing crash while deleting non-existing manifest entries. HT to zsfelfoldi

  * No leading slash in typical manifests.

  * Deleting images works

  * Adding images works.

  * Change tests to conform with new API. Semantics of tests not changed.

  * Additional error handling in roundtripper.

  * fix roundtripper test

swarm/api/api.go

@@ -52,11 +52,15 @@ func (self *Api) Store(data storage.SectionReader, wg *sync.WaitGroup) (key stor
 }
 
 // DNS Resolver
-func (self *Api) Resolve(hostPort string) (contentHash storage.Key, err error) {
+func (self *Api) Resolve(hostPort string, nameresolver bool) (contentHash storage.Key, err error) {
 	if hashMatcher.MatchString(hostPort) || self.dns == nil {
-		glog.V(logger.Debug).Infof("[BZZ] host is a contentHash: '%v'", contentHash)
+		glog.V(logger.Debug).Infof("[BZZ] host is a contentHash: '%v'", hostPort)
 		return storage.Key(common.Hex2Bytes(hostPort)), nil
 	}
+	if !nameresolver {
+		err = fmt.Errorf("'%s' is not a content hash value.", hostPort)
+		return
+	}
 	contentHash, err = self.dns.Resolve(hostPort)
 	if err != nil {
 		err = ErrResolve(err)
@@ -73,28 +77,27 @@ func parse(uri string) (hostPort, path string) {
 		return
 	}
 	// beginning with slash is now optional
-	if len(parts[0]) == 0 {
+	for len(parts[i]) == 0 {
 		i++
 	}
 	hostPort = parts[i]
-	if len(parts) > i+1 {
+	for i < len(parts)-1 {
-		path = parts[i+1]
+		i++
-		if len(parts) == 3 {
+		if len(path) > 0 {
-			path += "/" + parts[2]
+			path = path + "/" + parts[i]
+		} else {
+			path = parts[i]
 		}
-		path += "/"
-	}
-	if len(path) > 0 {
-		path = "/" + path
 	}
 	glog.V(logger.Debug).Infof("[BZZ] Swarm: host: '%s', path '%s' requested.", hostPort, path)
 	return
 }
 
-func (self *Api) parseAndResolve(uri string) (contentHash storage.Key, hostPort, path string, err error) {
+func (self *Api) parseAndResolve(uri string, nameresolver bool) (contentHash storage.Key, hostPort, path string, err error) {
 	hostPort, path = parse(uri)
 	//resolving host and port
-	contentHash, err = self.Resolve(hostPort)
+	contentHash, err = self.Resolve(hostPort, nameresolver)
+	glog.V(logger.Debug).Infof("[BZZ] Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path)
 	return
 }
 
@@ -119,9 +122,9 @@ func (self *Api) Put(content, contentType string) (string, error) {
 // Get uses iterative manifest retrieval and prefix matching
 // to resolve path to content using dpa retrieve
 // it returns a section reader, mimeType, status and an error
-func (self *Api) Get(uri string) (reader storage.SectionReader, mimeType string, status int, err error) {
+func (self *Api) Get(uri string, nameresolver bool) (reader storage.SectionReader, mimeType string, status int, err error) {
 
-	key, _, path, err := self.parseAndResolve(uri)
+	key, _, path, err := self.parseAndResolve(uri, nameresolver)
 
 	trie, err := loadManifest(self.dpa, key)
 	if err != nil {
@@ -144,8 +147,8 @@ func (self *Api) Get(uri string) (reader storage.SectionReader, mimeType string,
 	return
 }
 
-func (self *Api) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
+func (self *Api) Modify(uri, contentHash, contentType string, nameresolver bool) (newRootHash string, err error) {
-	root := common.Hex2Bytes(rootHash)
+	root, _, path, err := self.parseAndResolve(uri, nameresolver)
 	trie, err := loadManifest(self.dpa, root)
 	if err != nil {
 		return

swarm/api/api_test.go

@@ -63,7 +63,7 @@ func expResponse(content string, mimeType string, status int) *Response {
 
 // func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
 func testGet(t *testing.T, api *Api, bzzhash string) *testResponse {
-	reader, mimeType, status, err := api.Get(bzzhash)
+	reader, mimeType, status, err := api.Get(bzzhash, true)
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}

swarm/api/filesystem.go

@@ -167,7 +167,7 @@ func (self *FileSystem) Download(bzzpath, localpath string) error {
 	}
 
 	//resolving host and port
-	key, _, path, err := self.api.parseAndResolve(bzzpath)
+	key, _, path, err := self.api.parseAndResolve(bzzpath, true)
 	if err != nil {
 		return err
 	}

swarm/api/filesystem_test.go

@@ -58,7 +58,7 @@ func TestApiDirUpload0(t *testing.T) {
 		resp = testGet(t, api, bzzhash+"/img/logo.png")
 		exp = expResponse(content, "image/png", 0)
 
-		_, _, _, err = api.Get(bzzhash)
+		_, _, _, err = api.Get(bzzhash, true)
 		if err == nil {
 			t.Fatalf("expected error: %v", err)
 		}
@@ -91,17 +91,17 @@ func TestApiDirUploadModify(t *testing.T) {
 			return
 		}
 
-		bzzhash, err = api.Modify(bzzhash, "index.html", "", "")
+		bzzhash, err = api.Modify(bzzhash+"/index.html", "", "", true)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash, "index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8")
+		bzzhash, err = api.Modify(bzzhash+"/index2.html", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
 		}
-		bzzhash, err = api.Modify(bzzhash, "img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8")
+		bzzhash, err = api.Modify(bzzhash+"/img/logo.png", "9ea1f60ebd80786d6005f6b256376bdb494a82496cd86fe8c307cdfb23c99e71", "text/html; charset=utf-8", true)
 		if err != nil {
 			t.Errorf("unexpected error: %v", err)
 			return
@@ -120,7 +120,7 @@ func TestApiDirUploadModify(t *testing.T) {
 		resp = testGet(t, api, bzzhash+"/index.css")
 		exp = expResponse(content, "text/css", 0)
 
-		_, _, _, err = api.Get(bzzhash)
+		_, _, _, err = api.Get(bzzhash, true)
 		if err == nil {
 			t.Errorf("expected error: %v", err)
 		}

swarm/api/http/roundtripper.go

@@ -20,8 +20,8 @@ import (
 client := httpclient.New()
 // for (private) swarm proxy running locally
 client.RegisterScheme("bzz", &http.RoundTripper{Port: port})
-// for public swarm gateway
+client.RegisterScheme("bzzi", &http.RoundTripper{Port: port})
-client.RegisterScheme(scheme, &http.RoundTripper{Host: host, Port: port})
+client.RegisterScheme("bzzr", &http.RoundTripper{Port: port})
 
 The port you give the Roundtripper is the port the swarm proxy is listening on.
 If Host is left empty, localhost is assumed.
@@ -43,7 +43,11 @@ func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err
 	if len(host) == 0 {
 		host = "localhost"
 	}
-	url := fmt.Sprintf("http://%s:%s/%s/%s", host, self.Port, req.URL.Host, req.URL.Path)
+	url := fmt.Sprintf("http://%s:%s/%s:/%s/%s", host, self.Port, req.Proto, req.URL.Host, req.URL.Path)
 	glog.V(logger.Info).Infof("[BZZ] roundtripper: proxying request '%s' to '%s'", req.RequestURI, url)
-	return http.Get(url)
+	reqProxy, err := http.NewRequest(req.Method, url, req.Body)
+	if err != nil {
+		return nil, err
+	}
+	return http.DefaultClient.Do(reqProxy)
 }

swarm/api/http/roundtripper_test.go

@@ -43,8 +43,8 @@ func TestRoundTripper(t *testing.T) {
 		t.Errorf("expected no error, got %v", err)
 		return
 	}
-	if string(content) != "/test.com/path" {
+	if string(content) != "/HTTP/1.1:/test.com/path" {
-		t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/test.com/path", string(content))
+		t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/HTTP/1.1:/test.com/path", string(content))
 	}
 
 }

swarm/api/http/server.go

@@ -22,8 +22,8 @@ const (
 )
 
 var (
-	bzzPrefix       = regexp.MustCompile("^/+bzz:/+")
+	// accepted protocols: bzz (traditional), bzzi (immutable) and bzzr (raw)
-	rawUrl          = regexp.MustCompile("^/+raw/*")
+	bzzPrefix       = regexp.MustCompile("^/+bzz[ir]?:/+")
 	trailingSlashes = regexp.MustCompile("/+$")
 	// forever         = func() time.Time { return time.Unix(0, 0) }
 	forever = time.Now
@@ -62,20 +62,37 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 	//			return
 	//		}
 	//	}
-	glog.V(logger.Debug).Infof("[BZZ] Swarm: HTTP request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))
+	glog.V(logger.Debug).Infof("[BZZ] Swarm: HTTP %s request URL: '%s', Host: '%s', Path: '%s', Referer: '%s', Accept: '%s'", r.Method, r.RequestURI, requestURL.Host, requestURL.Path, r.Referer(), r.Header.Get("Accept"))
 	uri := requestURL.Path
-	var raw bool
+	var raw, nameresolver bool
+	var proto string
 
 	// HTTP-based URL protocol handler
-	uri = bzzPrefix.ReplaceAllString(uri, "")
 	glog.V(logger.Debug).Infof("[BZZ] Swarm: BZZ request URI: '%s'", uri)
 
-	path := rawUrl.ReplaceAllStringFunc(uri, func(string) string {
+	path := bzzPrefix.ReplaceAllStringFunc(uri, func(p string) string {
-		raw = true
+		proto = p
 		return ""
 	})
 
-	glog.V(logger.Debug).Infof("[BZZ] Swarm: %s request '%s' received.", r.Method, uri)
+	// protocol identification (ugly)
+	if proto == "" {
+		if glog.V(logger.Error) {
+			glog.Errorf(
+				"[BZZ] Swarm: Protocol error in request `%s`.",
+				uri,
+			)
+			http.Error(w, "BZZ protocol error", http.StatusBadRequest)
+			return
+		}
+	}
+	raw = proto[1:5] == "bzzr"
+	nameresolver = proto[1:5] != "bzzi"
+
+	glog.V(logger.Debug).Infof(
+		"[BZZ] Swarm: %s request over protocol %s '%s' received.",
+		r.Method, proto, path,
+	)
 
 	switch {
 	case r.Method == "POST" || r.Method == "PUT":
@@ -107,7 +124,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 				mime := r.Header.Get("Content-Type")
 				// TODO proper root hash separation
 				glog.V(logger.Debug).Infof("[BZZ] Modify '%s' to store %v as '%s'.", path, key.Log(), mime)
-				newKey, err := a.Modify(path[:64], path[65:], common.Bytes2Hex(key), mime)
+				newKey, err := a.Modify(path, common.Bytes2Hex(key), mime, nameresolver)
 				if err == nil {
 					glog.V(logger.Debug).Infof("[BZZ] Swarm replaced manifest by '%s'", newKey)
 					w.Header().Set("Content-Type", "text/plain")
@@ -125,7 +142,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		} else {
 			path = api.RegularSlashes(path)
 			glog.V(logger.Debug).Infof("[BZZ] Delete '%s'.", path)
-			newKey, err := a.Modify(path[:64], path[65:], "", "")
+			newKey, err := a.Modify(path, "", "", nameresolver)
 			if err == nil {
 				glog.V(logger.Debug).Infof("[BZZ] Swarm replaced manifest by '%s'", newKey)
 				w.Header().Set("Content-Type", "text/plain")
@@ -139,7 +156,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 		path = trailingSlashes.ReplaceAllString(path, "")
 		if raw {
 			// resolving host
-			key, err := a.Resolve(path)
+			key, err := a.Resolve(path, nameresolver)
 			if err != nil {
 				glog.V(logger.Error).Infof("[BZZ] Swarm: %v", err)
 				http.Error(w, err.Error(), http.StatusBadRequest)
@@ -166,7 +183,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) {
 
 			glog.V(logger.Debug).Infof("[BZZ] Swarm: Structured GET request '%s' received.", uri)
 
-			reader, mimeType, status, err := a.Get(path)
+			reader, mimeType, status, err := a.Get(path, nameresolver)
 			if err != nil {
 				if _, ok := err.(api.ErrResolve); ok {
 					glog.V(logger.Debug).Infof("[BZZ] Swarm: %v", err)

swarm/api/manifest.go

@@ -145,7 +145,10 @@ func (self *manifestTrie) deleteEntry(path string) {
 
 	b := byte(path[0])
 	entry := self.entries[b]
-	if (entry != nil) && (entry.Path == path) {
+	if entry == nil {
+		return
+	}
+	if entry.Path == path {
 		self.entries[b] = nil
 		return
 	}

swarm/api/storage.go

@@ -37,7 +37,7 @@ func (self *Storage) Put(content, contentType string) (string, error) {
 // the actual size of which is given in len(resp.Content), while the expected
 // size is resp.Size
 func (self *Storage) Get(bzzpath string) (*Response, error) {
-	reader, mimeType, status, err := self.api.Get(bzzpath)
+	reader, mimeType, status, err := self.api.Get(bzzpath, true)
 	if err != nil {
 		return nil, err
 	}
@@ -52,5 +52,5 @@ func (self *Storage) Get(bzzpath string) (*Response, error) {
 }
 
 func (self *Storage) Modify(rootHash, path, contentHash, contentType string) (newRootHash string, err error) {
-	return self.api.Modify(rootHash, path, contentHash, contentType)
+	return self.api.Modify(rootHash+"/"+path, contentHash, contentType, true)
 }

swarm/cmd/bzzup.sh

@@ -9,9 +9,9 @@ if [[ ! -z "$2" ]]; then
 fi
 
 if [ -f "$1" ]; then
-hash=`wget -q -O- --post-file="$1" http://localhost:$port/raw`
+hash=`wget -q -O- --post-file="$1" "http://localhost:$port/bzzr:/"`
 mime=`mimetype -b "$1"`
-wget -q -O- --post-data="$delimiter\"hash\":\"$hash\",\"contentType\":\"$mime\"}]}" http://localhost:8500/raw
+wget -q -O- --post-data="$delimiter\"hash\":\"$hash\",\"contentType\":\"$mime\"}]}" "http://localhost:$port/bzzr:/"
 echo
 
 else
@@ -28,7 +28,7 @@ do
 name=`echo "$path" | cut -c3-`
 [ _`basename "$name"` = "_$INDEX" ] && name=`dirname "$name"`
 echo -n "$delimiter"
-hash=`wget -q -O- --post-file="$path" http://localhost:$port/raw`
+hash=`wget -q -O- --post-file="$path" "http://localhost:$port/bzzr:/"`
 mime=`mimetype -b "$path"`
 if [ "_$name" = '_.' ]; then
 echo -n "\"hash\":\"$hash\",\"contentType\":\"$mime\""
@@ -38,7 +38,7 @@ fi
 delimiter='},{'
 
 done
-echo -n '}]}') | wget -q -O- --post-data=`cat` http://localhost:$port/raw
+echo -n '}]}') | wget -q -O- --post-data=`cat` "http://localhost:$port/bzzr:/"
 echo
 
 popd > /dev/null

swarm/examples/album/index.js

@@ -288,7 +288,7 @@ function uploadFile(files, nr, uri) {
       var xhr = new XMLHttpRequest();
       xhr.onreadystatechange = function() { if (xhr.readyState === 4) {
         var i = xhr.responseText;
-        window.location.replace("/" + i + "/");
+        window.location.replace("/bzz:/" + i + "/");
       }};
       sendImgs(xhr, uri);
     }
@@ -327,9 +327,9 @@ function uploadFile(files, nr, uri) {
       imgData[0] = "imgs/" + file.name;
       imgData[1] = [img.naturalWidth, img.naturalHeight];
       imgs.data.splice(eidx, 0, {img: imgData, thumb: thumbData, blur: blur});
-      uploadFile(files, nr + 1, "/" + i + "/");
+      uploadFile(files, nr + 1, "/bzz:/" + i + "/");
     }
-    img.src = "/" + i + "/imgs/" + file.name;
+    img.src = "/bzz:/" + i + "/imgs/" + file.name;
     return;
   }};
   xhr.open("PUT", uri + "imgs/" + file.name, true);
@@ -361,9 +361,9 @@ function deleteImg()
     var xhrd = new XMLHttpRequest();
     xhrd.onreadystatechange = function () {  if (xhrd.readyState === 4) {
       var j = xhrd.responseText;
-      window.location.replace("/" + j + "/");
+      window.location.replace("/bzz:/" + j + "/");
     }};
-    xhrd.open("DELETE", "/" + i + "/" + fname, true);
+    xhrd.open("DELETE", "/bzz%3A/" + i + "/" + fname, true);
     xhrd.send();
   }};
 
@@ -378,7 +378,7 @@ function moveUpDown(off)
   var xhr = new XMLHttpRequest();
   xhr.onreadystatechange = function () {  if (xhr.readyState === 4) {
     var i = xhr.responseText;
-    window.location.replace("/" + i + "/#" + (eidx + off));
+    window.location.replace("/bzz:/" + i + "/#" + (eidx + off));
   }};
   sendImgs(xhr, "");
 }

swarm/examples/bzzhandler.html

@@ -6,10 +6,21 @@
     navigator.registerProtocolHandler("bzz",
                                   "http://localhost:8500/%s",
                                   "Swarm handler");
+    navigator.registerProtocolHandler("bzzi",
+                                  "http://localhost:8500/%s",
+                                  "Swarm immutable handler");
+    navigator.registerProtocolHandler("bzzr",
+                                  "http://localhost:8500/%s",
+                                  "Swarm raw handler");
   </script>
 </head>
 <body>
   <h1>Register Swarm protocol handler</h1>
-  <p>This web page will install a web protocol handler for the <code>bzz:</code> protocol.</p>
+  <p>This web page will install a web protocol handler for
+    <code>bzz:</code>, 
+    <code>bzzi:</code> and 
+    <code>bzzr:</code> 
+    protocols.
+  </p>
 </body>
 </html>