From 7f50f8d54e6ff1eb92eada66ed2e19302c31ac08 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 27 May 2015 11:03:33 +0100 Subject: [PATCH] http: url handling improved improved logs --- bzz/api.go | 41 +++++++++++++++++++++++++++++------------ bzz/chunker.go | 10 +++++----- bzz/dpa.go | 5 +++-- bzz/httpaccess.go | 33 +++++++++++++++++---------------- bzz/manifest.go | 19 +++++++------------ 5 files changed, 61 insertions(+), 47 deletions(-) diff --git a/bzz/api.go b/bzz/api.go index 550743b349..675ab4f4dd 100644 --- a/bzz/api.go +++ b/bzz/api.go @@ -5,7 +5,7 @@ import ( "fmt" "net" "path/filepath" - "strings" + "regexp" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/resolver" @@ -14,6 +14,11 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" ) +var ( + hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}") + slashes = regexp.MustCompile("/+") +) + /* Api implements webserver/file system related content storage and retrieval on top of the dpa @@ -50,6 +55,7 @@ func (self *Api) Bzz() (p2p.Protocol, error) { func (self *Api) Start(node *discover.Node, connectPeer func(string) error) { self.dpa.Start() self.netStore.Start(node, connectPeer) + dpaLogger.Infof("Swarm started.") go startHttpServer(self, self.port) } @@ -89,15 +95,20 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve) var host, port string var err error host, port, err = net.SplitHostPort(hostport) - if err != nil && err.Error() != "missing port in address "+hostport { - errR = errResolve(fmt.Errorf("invalid host '%s': %v", hostport, err)) - return + if err != nil { + if err.Error() == "missing port in address "+hostport { + host = hostport + } else { + errR = errResolve(fmt.Errorf("invalid host '%s': %v", hostport, err)) + return + } } if hashMatcher.MatchString(host) { - contentHash = Key(host) + contentHash = Key(common.Hex2Bytes(host)) + dpaLogger.Debugf("Swarm host is a contentHash: '%064x'", contentHash) } else { if self.Resolver != nil { - hostHash := common.BytesToHash(crypto.Sha3([]byte(host))) + hostHash := common.BytesToHash(crypto.Sha3(common.Hex2Bytes(host))) // TODO: should take port as block number versioning _ = port var hash common.Hash @@ -106,6 +117,7 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve) err = errResolve(fmt.Errorf("unable to resolve '%s': %v", hostport, err)) } contentHash = Key(hash.Bytes()) + dpaLogger.Debugf("Swarm resolve host to contentHash: '%064x'", contentHash) } else { err = errResolve(fmt.Errorf("no resolver '%s': %v", hostport, err)) } @@ -114,9 +126,12 @@ func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve) } func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, status int, err error) { - parts := strings.SplitAfterN(uri[1:], "/", 2) - hostPort := parts[0] - path := parts[1] + parts := slashes.Split(uri, 3) + hostPort := parts[1] + var path string + if len(parts) > 2 { + path = parts[2] + } dpaLogger.Debugf("Swarm: host: '%s', path '%s' requested.", hostPort, path) //resolving host and port @@ -129,6 +144,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta // retrieve content following path along manifests var pos int for { + dpaLogger.Debugf("Swarm: manifest lookup key: '%064x'.", key) // retrieve manifest via DPA manifestReader := self.dpa.Retrieve(key) // TODO check size for oversized manifests @@ -143,7 +159,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta return } - dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved.", uri) + dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved", uri) man := manifest{} err = json.Unmarshal(manifestData, &man) if err != nil { @@ -152,7 +168,7 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta return } - dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries.", uri, len(man.Entries)) + dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries. Retrieving entry for '%s'", uri, len(man.Entries), path) // retrieve entry that matches path from manifest entries var entry *manifestEntry @@ -172,13 +188,14 @@ func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, sta // get mime type of entry mimeType = entry.ContentType - if mimeType != "" { + if mimeType == "" { mimeType = manifestType } // if path matched on non-manifest content type, then retrieve reader // and return if mimeType != manifestType { + dpaLogger.Debugf("Swarm: content lookup key: '%064x' (%v)", key, mimeType) reader = self.dpa.Retrieve(key) return } diff --git a/bzz/chunker.go b/bzz/chunker.go index b63a0d43b3..db0520536b 100644 --- a/bzz/chunker.go +++ b/bzz/chunker.go @@ -309,13 +309,13 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { C: make(chan bool), // close channel to signal data delivery } self.chunkC <- chunk // submit retrieval request, someone should be listening on the other side (or we will time out globally) - dpaLogger.Debugf("readAt %x into %d bytes at %d.", chunk.Key[:4], len(b), off) + dpaLogger.Debugf("readAt: reading %x into %d bytes at offset %d.", chunk.Key[:4], len(b), off) // waiting for the chunk retrieval select { case <-self.quitC: // this is how we control process leakage (quitC is closed once join is finished (after timeout)) - dpaLogger.Debugf("quit") + // dpaLogger.Debugf("quit") return case <-chunk.C: // bells are ringing, data have been delivered // dpaLogger.Debugf("chunk data received for %x", chunk.Key[:4]) @@ -355,9 +355,9 @@ func (self *LazyChunkReader) ReadAt(b []byte, off int64) (read int, err error) { if off+int64(read) == self.size { err = io.EOF } - dpaLogger.Debugf("ReadAt returning with %d, %v", read, err) + dpaLogger.Debugf("ReadAt returning at %d: %v", read, err) case <-self.quitC: - dpaLogger.Debugf("ReadAt aborted with %d, %v", read, err) + dpaLogger.Debugf("ReadAt aborted at %d: %v", read, err) } return } @@ -422,7 +422,7 @@ func (self *LazyChunkReader) join(b []byte, off int64, eoff int64, depth int, tr // this is how we control process leakage (quitC is closed once join is finished (after timeout)) return case <-ch.C: // bells are ringing, data have been delivered - dpaLogger.Debugf("chunk data received") + // dpaLogger.Debugf("chunk data received") } if soff < off { soff = off diff --git a/bzz/dpa.go b/bzz/dpa.go index 401585b087..e43b9ac646 100644 --- a/bzz/dpa.go +++ b/bzz/dpa.go @@ -84,7 +84,7 @@ SPLIT: select { case err, ok := <-errC: if err != nil { - dpaLogger.Warnf("chunkner split error: %v", err) + dpaLogger.Warnf("chunker split error: %v", err) } if !ok { break SPLIT @@ -108,7 +108,7 @@ func (self *DPA) Start() { self.quitC = make(chan bool) self.storeLoop() self.retrieveLoop() - dpaLogger.Infof("Swarm started.") + dpaLogger.Debugf("Swarm DPA started.") } func (self *DPA) Stop() { @@ -129,6 +129,7 @@ func (self *DPA) retrieveLoop() { for ch := range self.retrieveC { go func(chunk *Chunk) { + dpaLogger.DebugDetailf("dpa: retrieve loop : chunk '%x'", chunk.Key) storedChunk, err := self.ChunkStore.Get(chunk.Key) if err == notFound { dpaLogger.DebugDetailf("chunk '%x' not found", chunk.Key) diff --git a/bzz/httpaccess.go b/bzz/httpaccess.go index 701e8d5797..38534e6fd4 100644 --- a/bzz/httpaccess.go +++ b/bzz/httpaccess.go @@ -18,10 +18,8 @@ const ( ) var ( - // protocolMatcher = regexp.MustCompile("^/bzz:") - rawManifestMatcher = regexp.MustCompile("^/raw/") - // rawManifestMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}(?:/[a-z]+/[-+0-9a-z]+)?$") - manifestMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}") + rawUrl = regexp.MustCompile("^/+raw/*") + trailingSlashes = regexp.MustCompile("/+$") ) type sequentialReader struct { @@ -43,11 +41,17 @@ func startHttpServer(api *Api, port string) { func handler(w http.ResponseWriter, r *http.Request, api *Api) { dpaLogger.Debugf("request URL: '%s' Host: '%s', Path: '%s'", r.RequestURI, r.URL.Host, r.URL.Path) + uri := r.URL.Path + var raw bool + path := rawUrl.ReplaceAllStringFunc(uri, func(string) string { + raw = true + return "" + }) switch { case r.Method == "POST": dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, r.URL.Path) - if r.URL.Path == "/raw" { + if raw { dpaLogger.Debugf("Swarm: POST request received.") key, err := api.dpa.Store(io.NewSectionReader(&sequentialReader{ reader: r.Body, @@ -61,20 +65,18 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) { return } } else { - http.Error(w, "No POST to "+r.URL.Path+" allowed.", http.StatusBadRequest) + http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest) return } case r.Method == "GET" || r.Method == "HEAD": - uri := r.URL.Path - dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, r.URL.Path) - // raw , - if rawManifestMatcher.MatchString(uri) { + dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, uri) + path = trailingSlashes.ReplaceAllString(path, "") + if raw { dpaLogger.Debugf("Swarm: Raw GET request '%s' received", uri) // resolving host - name := uri[5:] - key, err := api.resolveHost(name) + key, err := api.resolveHost(path) if err != nil { dpaLogger.Debugf("Swarm: %v", err) http.Error(w, err.Error(), http.StatusBadRequest) @@ -93,7 +95,7 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) { } w.Header().Set("Content-Type", mimeType) - http.ServeContent(w, r, name, time.Unix(0, 0), reader) + http.ServeContent(w, r, uri, time.Unix(0, 0), reader) dpaLogger.Debugf("Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType) // retrieve path via manifest @@ -102,9 +104,8 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) { dpaLogger.Debugf("Swarm: Structured GET request '%s' received.", uri) // call to api.getPath on uri - reader, mimeType, status, err := api.getPath(uri[1:]) + reader, mimeType, status, err := api.getPath(path) if err != nil { - var status int if _, ok := err.(errResolve); ok { dpaLogger.Debugf("Swarm: %v", err) status = http.StatusBadRequest @@ -121,8 +122,8 @@ func handler(w http.ResponseWriter, r *http.Request, api *Api) { if status > 0 { w.WriteHeader(status) } + dpaLogger.Debugf("Swarm: Served '%s' (%d bytes) as '%s' (status code: %v)", uri, reader.Size(), mimeType, w.Header()) http.ServeContent(w, r, uri, time.Unix(0, 0), reader) - dpaLogger.Debugf("Swarm: Served '%s' (%d bytes) as '%s' (status code: %d)", uri, reader.Size(), mimeType, status) } default: diff --git a/bzz/manifest.go b/bzz/manifest.go index fcd04ea83e..6416530e3e 100644 --- a/bzz/manifest.go +++ b/bzz/manifest.go @@ -1,34 +1,29 @@ package bzz import ( - // "fmt" - "regexp" +// "fmt" ) const ( manifestType = "application/bzz-manifest+json" ) -var ( - hashMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}") -) - type manifest struct { - Entries []*manifestEntry `"json:entries"` + Entries []*manifestEntry `json:"entries"` } type manifestEntry struct { - Path string `"json:path"` - Hash string `"json:hash"` - ContentType string `"json:contentType"` - Status int `"json:status"` + Path string `json:"path"` + Hash string `json:"hash"` + ContentType string `json:"contentType"` + Status int `json:"status"` } func (self *manifest) getEntry(path string) (entry *manifestEntry, pos int) { for _, entry = range self.Entries { pos = len(entry.Path) if len(path) >= pos && path[:pos] == entry.Path { - dpaLogger.Debugf("Swarm: \"%s\" matches \"%s\".", path, entry.Path) + dpaLogger.Debugf("Swarm: '%s' matches '%s'.", path, entry.Path) return } }