http: url handling improved

improved logs

bzz/api.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"net"
 	"path/filepath"
-	"strings"
+	"regexp"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/resolver"
@@ -14,6 +14,11 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/discover"
 )
 
+var (
+	hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}")
+	slashes     = regexp.MustCompile("/+")
+)
+
 /*
 Api implements webserver/file system related content storage and retrieval
 on top of the dpa
@@ -50,6 +55,7 @@ func (self *Api) Bzz() (p2p.Protocol, error) {
 func (self *Api) Start(node *discover.Node, connectPeer func(string) error) {
 	self.dpa.Start()
 	self.netStore.Start(node, connectPeer)
+	dpaLogger.Infof("Swarm started.")
 	go startHttpServer(self, self.port)
 }
 
@@ -89,15 +95,20 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve)
 	var host, port string
 	var err error
 	host, port, err = net.SplitHostPort(hostport)
-	if err != nil && err.Error() != "missing port in address "+hostport {
+	if err != nil {
-		errR = errResolve(fmt.Errorf("invalid host '%s': %v", hostport, err))
+		if err.Error() == "missing port in address "+hostport {
-		return
+			host = hostport
+		} else {
+			errR = errResolve(fmt.Errorf("invalid host '%s': %v", hostport, err))
+			return
+		}
 	}
 	if hashMatcher.MatchString(host) {
-		contentHash = Key(host)
+		contentHash = Key(common.Hex2Bytes(host))
+		dpaLogger.Debugf("Swarm host is a contentHash: '%064x'", contentHash)
 	} else {
 		if self.Resolver != nil {
-			hostHash := common.BytesToHash(crypto.Sha3([]byte(host)))
+			hostHash := common.BytesToHash(crypto.Sha3(common.Hex2Bytes(host)))
 			// TODO: should take port as block number versioning
 			_ = port
 			var hash common.Hash
@@ -106,6 +117,7 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve)
 				err = errResolve(fmt.Errorf("unable to resolve '%s': %v", hostport, err))
 			}
 			contentHash = Key(hash.Bytes())
+			dpaLogger.Debugf("Swarm resolve host to contentHash: '%064x'", contentHash)
 		} else {
 			err = errResolve(fmt.Errorf("no resolver '%s': %v", hostport, err))
 		}
@@ -114,9 +126,12 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve)
 }
 
 func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, status int, err error) {
-	parts := strings.SplitAfterN(uri[1:], "/", 2)
+	parts := slashes.Split(uri, 3)
-	hostPort := parts[0]
+	hostPort := parts[1]
-	path := parts[1]
+	var path string
+	if len(parts) > 2 {
+		path = parts[2]
+	}
 	dpaLogger.Debugf("Swarm: host: '%s', path '%s' requested.", hostPort, path)
 
 	//resolving host and port
@@ -129,6 +144,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta
 	// retrieve content following path along manifests
 	var pos int
 	for {
+		dpaLogger.Debugf("Swarm: manifest lookup key: '%064x'.", key)
 		// retrieve manifest via DPA
 		manifestReader := self.dpa.Retrieve(key)
 		// TODO check size for oversized manifests
@@ -143,7 +159,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta
 			return
 		}
 
-		dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved.", uri)
+		dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved", uri)
 		man := manifest{}
 		err = json.Unmarshal(manifestData, &man)
 		if err != nil {
@@ -152,7 +168,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta
 			return
 		}
 
-		dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries.", uri, len(man.Entries))
+		dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries. Retrieving entry for '%s'", uri, len(man.Entries), path)
 
 		// retrieve entry that matches path from manifest entries
 		var entry *manifestEntry
@@ -172,13 +188,14 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta
 
 		// get mime type of entry
 		mimeType = entry.ContentType
-		if mimeType != "" {
+		if mimeType == "" {
 			mimeType = manifestType
 		}
 
 		// if path matched on non-manifest content type, then retrieve reader
 		// and return
 		if mimeType != manifestType {
+			dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType)
 			reader = self.dpa.Retrieve(key)
 			return
 		}

bzz/chunker.go

@@ -309,13 +309,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
 		C:   make(chan bool), // close channel to signal data delivery
 	}
 	self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally)
-	dpaLogger.Debugf("readAt %x into %d bytes at %d.", chunk.Key[:4], len(b), off)
+	dpaLogger.Debugf("readAt: reading %x into %d bytes at offset %d.", chunk.Key[:4], len(b), off)
 
 	// waiting for the chunk retrieval
 	select {
 	case <-self.quitC:
 		// this is how we control process leakage (quitC is closed once join is finished (after timeout))
-		dpaLogger.Debugf("quit")
+		// dpaLogger.Debugf("quit")
 		return
 	case <-chunk.C: // bells are ringing, data have been delivered
 		// dpaLogger.Debugf("chunk data received for %x", chunk.Key[:4])
@@ -355,9 +355,9 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) {
 		if off+int64(read) == self.size {
 			err = io.EOF
 		}
-		dpaLogger.Debugf("ReadAt returning with %d, %v", read, err)
+		dpaLogger.Debugf("ReadAt returning at %d: %v", read, err)
 	case <-self.quitC:
-		dpaLogger.Debugf("ReadAt aborted with %d, %v", read, err)
+		dpaLogger.Debugf("ReadAt aborted at %d: %v", read, err)
 	}
 	return
 }
@@ -422,7 +422,7 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr
 				// this is how we control process leakage (quitC is closed once join is finished (after timeout))
 				return
 			case <-ch.C: // bells are ringing, data have been delivered
-				dpaLogger.Debugf("chunk data received")
+				// dpaLogger.Debugf("chunk data received")
 			}
 			if soff < off {
 				soff = off

bzz/dpa.go

@@ -84,7 +84,7 @@ SPLIT:
 		select {
 		case err, ok := <-errC:
 			if err != nil {
-				dpaLogger.Warnf("chunkner split error: %v", err)
+				dpaLogger.Warnf("chunker split error: %v", err)
 			}
 			if !ok {
 				break SPLIT
@@ -108,7 +108,7 @@ func (self *DPA) Start() {
 	self.quitC = make(chan bool)
 	self.storeLoop()
 	self.retrieveLoop()
-	dpaLogger.Infof("Swarm started.")
+	dpaLogger.Debugf("Swarm DPA started.")
 }
 
 func (self *DPA) Stop() {
@@ -129,6 +129,7 @@ func (self *DPA) retrieveLoop() {
 		for ch := range self.retrieveC {
 
 			go func(chunk *Chunk) {
+				dpaLogger.DebugDetailf("dpa: retrieve loop : chunk '%x'", chunk.Key)
 				storedChunk, err := self.ChunkStore.Get(chunk.Key)
 				if err == notFound {
 					dpaLogger.DebugDetailf("chunk '%x' not found", chunk.Key)

bzz/httpaccess.go

@@ -18,10 +18,8 @@ const (
 )
 
 var (
-	// protocolMatcher    = regexp.MustCompile("^/bzz:")
+	rawUrl          = regexp.MustCompile("^/+raw/*")
-	rawManifestMatcher = regexp.MustCompile("^/raw/")
+	trailingSlashes = regexp.MustCompile("/+$")
-	// rawManifestMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}(?:/[a-z]+/[-+0-9a-z]+)?$")
-	manifestMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}")
 )
 
 type sequentialReader struct {
@@ -43,11 +41,17 @@ func startHttpServer(api *Api, port string) {
 
 func handler(w http.ResponseWriter, r *http.Request, api *Api) {
 	dpaLogger.Debugf("request URL: '%s' Host: '%s', Path: '%s'", r.RequestURI, r.URL.Host, r.URL.Path)
+	uri := r.URL.Path
+	var raw bool
+	path := rawUrl.ReplaceAllStringFunc(uri, func(string) string {
+		raw = true
+		return ""
+	})
 
 	switch {
 	case r.Method == "POST":
 		dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, r.URL.Path)
-		if r.URL.Path == "/raw" {
+		if raw {
 			dpaLogger.Debugf("Swarm: POST request received.")
 			key, err := api.dpa.Store(io.NewSectionReader(&sequentialReader{
 				reader: r.Body,
@@ -61,20 +65,18 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) {
 				return
 			}
 		} else {
-			http.Error(w, "No POST to "+r.URL.Path+" allowed.", http.StatusBadRequest)
+			http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest)
 			return
 		}
 
 	case r.Method == "GET" || r.Method == "HEAD":
-		uri := r.URL.Path
+		dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, uri)
-		dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, r.URL.Path)
+		path = trailingSlashes.ReplaceAllString(path, "")
-		// raw ,
+		if raw {
-		if rawManifestMatcher.MatchString(uri) {
 			dpaLogger.Debugf("Swarm: Raw GET request '%s' received", uri)
 
 			// resolving host
-			name := uri[5:]
+			key, err := api.resolveHost(path)
-			key, err := api.resolveHost(name)
 			if err != nil {
 				dpaLogger.Debugf("Swarm: %v", err)
 				http.Error(w, err.Error(), http.StatusBadRequest)
@@ -93,7 +95,7 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) {
 			}
 
 			w.Header().Set("Content-Type", mimeType)
-			http.ServeContent(w, r, name, time.Unix(0, 0), reader)
+			http.ServeContent(w, r, uri, time.Unix(0, 0), reader)
 			dpaLogger.Debugf("Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType)
 
 			// retrieve path via manifest
@@ -102,9 +104,8 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) {
 			dpaLogger.Debugf("Swarm: Structured GET request '%s' received.", uri)
 
 			// call to api.getPath on uri
-			reader, mimeType, status, err := api.getPath(uri[1:])
+			reader, mimeType, status, err := api.getPath(path)
 			if err != nil {
-				var status int
 				if _, ok := err.(errResolve); ok {
 					dpaLogger.Debugf("Swarm: %v", err)
 					status = http.StatusBadRequest
@@ -121,8 +122,8 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) {
 			if status > 0 {
 				w.WriteHeader(status)
 			}
+			dpaLogger.Debugf("Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, w.Header())
 			http.ServeContent(w, r, uri, time.Unix(0, 0), reader)
-			dpaLogger.Debugf("Swarm: Served '%s' (%d bytes) as '%s' (status code: %d)", uri, reader.Size(), mimeType, status)
 
 		}
 	default:

bzz/manifest.go

@@ -1,34 +1,29 @@
 package bzz
 
 import (
-	// "fmt"
+// "fmt"
-	"regexp"
 )
 
 const (
 	manifestType = "application/bzz-manifest+json"
 )
 
-var (
-	hashMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}")
-)
-
 type manifest struct {
-	Entries []*manifestEntry `"json:entries"`
+	Entries []*manifestEntry `json:"entries"`
 }
 
 type manifestEntry struct {
-	Path        string `"json:path"`
+	Path        string `json:"path"`
-	Hash        string `"json:hash"`
+	Hash        string `json:"hash"`
-	ContentType string `"json:contentType"`
+	ContentType string `json:"contentType"`
-	Status      int    `"json:status"`
+	Status      int    `json:"status"`
 }
 
 func (self *manifest) getEntry(path string) (entry *manifestEntry, pos int) {
 	for _, entry = range self.Entries {
 		pos = len(entry.Path)
 		if len(path) >= pos && path[:pos] == entry.Path {
-			dpaLogger.Debugf("Swarm: \"%s\" matches \"%s\".", path, entry.Path)
+			dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, entry.Path)
 			return
 		}
 	}