From fc8358a261b577ee0f5d29d6e01cb1637f0be244 Mon Sep 17 00:00:00 2001 From: "Daniel A. Nagy" Date: Tue, 10 Feb 2015 23:34:41 +0100 Subject: [PATCH 1/4] Interpreting manifest files (no recursion). --- bzz/httpaccess.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/bzz/httpaccess.go b/bzz/httpaccess.go index 824667596f..192cea7767 100644 --- a/bzz/httpaccess.go +++ b/bzz/httpaccess.go @@ -4,6 +4,7 @@ A simple http server interface to Swarm package bzz import ( + "encoding/json" "fmt" "github.com/ethereum/go-ethereum/ethutil" "io" @@ -17,7 +18,9 @@ const ( ) var ( - uriMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}$") + uriMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}$") + manifestMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}") + hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}$") ) type sequentialReader struct { @@ -26,6 +29,13 @@ type sequentialReader struct { ahead map[int64](chan bool) } +type manifestEntry struct { + Path string + Hash string + Content_type string + Status int16 +} + func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) { if self.pos != off { dpaLogger.Debugf("Swarm: deferred read in POST at position %d, offset %d.", @@ -83,6 +93,47 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { name := uri[5:] key := ethutil.Hex2Bytes(name) http.ServeContent(w, r, name+".bin", time.Unix(0, 0), dpa.Retrieve(key)) + } else if manifestMatcher.MatchString(uri) { + name := uri[1:65] + path := uri[65:] + key := ethutil.Hex2Bytes(name) + manifestReader := dpa.Retrieve(key) + // TODO check size for oversized manifests + manifest := make([]byte, manifestReader.Size()) + manifestReader.Read(manifest) + manifestEntries := make([]manifestEntry, 0) + json.Unmarshal(manifest, &manifestEntries) + var hash []byte + var mimeType string + prefix := 0 + status := int16(404) + for i, entry := range manifestEntries { + if !hashMatcher.MatchString(entry.Hash) { + // hash is mandatory + break + } + if entry.Content_type == "" { + // content type defaults to manifest + entry.Content_type = "application/bzz-manifest+json" + } + if entry.Status == 0 { + // status defaults to 200 + entry.Status = 200 + } + pathLen := len(entry.Path) + if len(path) >= pathLen && path[:pathLen] == entry.Path && prefix < pathLen { + prefix = pathLen + hash = ethutil.Hex2Bytes(entry.Hash) + mimeType = entry.Content_type + status = entry.Status + } + } + if hash == nil { + http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) + } else { + w.Header().Set("Content-Type", mimeType) + http.ServeContent(w, r, "", time.Unix(0, 0), dpa.Retrieve(hash)) + } } else { http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) } From 304ddcf562a74c0a352758b2f17db5cc615a5bd2 Mon Sep 17 00:00:00 2001 From: "Daniel A. Nagy" Date: Tue, 10 Feb 2015 23:45:27 +0100 Subject: [PATCH 2/4] Reviewed by Fefe --- bzz/httpaccess.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/bzz/httpaccess.go b/bzz/httpaccess.go index 192cea7767..ab13c8ca1d 100644 --- a/bzz/httpaccess.go +++ b/bzz/httpaccess.go @@ -95,7 +95,7 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { http.ServeContent(w, r, name+".bin", time.Unix(0, 0), dpa.Retrieve(key)) } else if manifestMatcher.MatchString(uri) { name := uri[1:65] - path := uri[65:] + path := uri[65:] // typically begins with a / key := ethutil.Hex2Bytes(name) manifestReader := dpa.Retrieve(key) // TODO check size for oversized manifests @@ -103,8 +103,8 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { manifestReader.Read(manifest) manifestEntries := make([]manifestEntry, 0) json.Unmarshal(manifest, &manifestEntries) - var hash []byte var mimeType string + key = nil prefix := 0 status := int16(404) for i, entry := range manifestEntries { @@ -123,16 +123,17 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { pathLen := len(entry.Path) if len(path) >= pathLen && path[:pathLen] == entry.Path && prefix < pathLen { prefix = pathLen - hash = ethutil.Hex2Bytes(entry.Hash) + key = ethutil.Hex2Bytes(entry.Hash) mimeType = entry.Content_type status = entry.Status } } - if hash == nil { + if key == nil { http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) } else { w.Header().Set("Content-Type", mimeType) - http.ServeContent(w, r, "", time.Unix(0, 0), dpa.Retrieve(hash)) + w.WriteHeader(status) + http.ServeContent(w, r, "", time.Unix(0, 0), dpa.Retrieve(key)) } } else { http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) From afcc7c4e79aad7ff5d7e4051bb4d4341cd0bce47 Mon Sep 17 00:00:00 2001 From: "Daniel A. Nagy" Date: Wed, 11 Feb 2015 00:21:58 +0100 Subject: [PATCH 3/4] Debugging and error handling extended. --- bzz/httpaccess.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/bzz/httpaccess.go b/bzz/httpaccess.go index ab13c8ca1d..35e84f29d6 100644 --- a/bzz/httpaccess.go +++ b/bzz/httpaccess.go @@ -90,19 +90,35 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { } case r.Method == "GET": if uriMatcher.MatchString(uri) { + dpaLogger.Debugf("Swarm: Raw GET request %s received", uri) name := uri[5:] key := ethutil.Hex2Bytes(name) http.ServeContent(w, r, name+".bin", time.Unix(0, 0), dpa.Retrieve(key)) + dpaLogger.Debugf("Swarm: Object %s returned.", name) } else if manifestMatcher.MatchString(uri) { + dpaLogger.Debugf("Swarm: Structured GET request %s received.", uri) name := uri[1:65] path := uri[65:] // typically begins with a / key := ethutil.Hex2Bytes(name) manifestReader := dpa.Retrieve(key) // TODO check size for oversized manifests manifest := make([]byte, manifestReader.Size()) - manifestReader.Read(manifest) + n, err := manifestReader.Read(manifest) + if err != nil { + dpaLogger.Debugf("Swarm: Manifest %s not found.", name) + http.Error(w, err.Error(), http.StatusNotFound) + return + } + dpaLogger.Debugf("Swarm: Manifest %s retrieved.") manifestEntries := make([]manifestEntry, 0) - json.Unmarshal(manifest, &manifestEntries) + err = json.Unmarshal(manifest, &manifestEntries) + if err != nil { + dpaLogger.Debugf("Swarm: Manifest %s is malformed.", name) + http.Error(w, err.Error(), http.StatusNotFound) + return + } else { + dpaLogger.Debugf("Swarm: Manifest %s has %d entries.", name, len(manifestEntries)) + } var mimeType string key = nil prefix := 0 @@ -132,8 +148,9 @@ func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) } else { w.Header().Set("Content-Type", mimeType) - w.WriteHeader(status) + w.WriteHeader(int(status)) http.ServeContent(w, r, "", time.Unix(0, 0), dpa.Retrieve(key)) + dpaLogger.Debugf("Swarm: Served %s as %s.", mimeType, uri) } } else { http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) From 9bbdcd0f3f3e340a03e8f1de3b4f938280d1cb8b Mon Sep 17 00:00:00 2001 From: "Daniel A. Nagy" Date: Wed, 11 Feb 2015 11:43:43 +0100 Subject: [PATCH 4/4] Serving http requests in a separate goroutine. --- eth/backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/backend.go b/eth/backend.go index 367ca7ec68..0e51f3a277 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -148,7 +148,7 @@ func New(config *Config) (*Ethereum, error) { ChunkStore: netStore, } dpa.Start() - bzz.StartHttpServer(dpa) + go bzz.StartHttpServer(dpa) nat, err := p2p.ParseNAT(config.NATType, config.PMPGateway) if err != nil {