Merge pull request #15 from ethereum/master

Pull from go-ethereum and merge

cmd/swarm/config.go

@@ -68,6 +68,7 @@ const (
 	SWARM_ENV_SWAP_API             = "SWARM_SWAP_API"
 	SWARM_ENV_SYNC_DISABLE         = "SWARM_SYNC_DISABLE"
 	SWARM_ENV_SYNC_UPDATE_DELAY    = "SWARM_ENV_SYNC_UPDATE_DELAY"
+	SWARM_ENV_LIGHT_NODE_ENABLE    = "SWARM_LIGHT_NODE_ENABLE"
 	SWARM_ENV_DELIVERY_SKIP_CHECK  = "SWARM_DELIVERY_SKIP_CHECK"
 	SWARM_ENV_ENS_API              = "SWARM_ENS_API"
 	SWARM_ENV_ENS_ADDR             = "SWARM_ENS_ADDR"
@@ -204,6 +205,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
 		currentConfig.SyncUpdateDelay = d
 	}
 
+	if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
+		currentConfig.LightNodeEnabled = true
+	}
+
 	if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
 		currentConfig.DeliverySkipCheck = true
 	}
@@ -301,6 +306,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
 		}
 	}
 
+	if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
+		if lightnode, err := strconv.ParseBool(lne); err != nil {
+			currentConfig.LightNodeEnabled = lightnode
+		}
+	}
+
 	if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
 		currentConfig.SwapAPI = swapapi
 	}

cmd/swarm/main.go

@@ -123,6 +123,11 @@ var (
 		Usage:  "Duration for sync subscriptions update after no new peers are added (default 15s)",
 		EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
 	}
+	SwarmLightNodeEnabled = cli.BoolFlag{
+		Name:   "lightnode",
+		Usage:  "Enable Swarm LightNode (default false)",
+		EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
+	}
 	SwarmDeliverySkipCheckFlag = cli.BoolFlag{
 		Name:   "delivery-skip-check",
 		Usage:  "Skip chunk delivery check (default false)",
@@ -464,6 +469,7 @@ pv(1) tool to get a progress bar:
 		SwarmSwapAPIFlag,
 		SwarmSyncDisabledFlag,
 		SwarmSyncUpdateDelay,
+		SwarmLightNodeEnabled,
 		SwarmDeliverySkipCheckFlag,
 		SwarmListenAddrFlag,
 		SwarmPortFlag,

p2p/discover/table.go

@@ -160,7 +160,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
 
 	// Find all non-empty buckets and get a fresh slice of their entries.
 	var buckets [][]*Node
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if len(b.entries) > 0 {
 			buckets = append(buckets, b.entries[:])
 		}
@@ -508,7 +508,7 @@ func (tab *Table) copyLiveNodes() {
 	defer tab.mutex.Unlock()
 
 	now := time.Now()
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			if now.Sub(n.addedAt) >= seedMinTableTime {
 				tab.db.updateNode(n)
@@ -524,7 +524,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 	// obviously correct. I believe that tree-based buckets would make
 	// this easier to implement efficiently.
 	close := &nodesByDistance{target: target}
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			close.push(n, nresults)
 		}
@@ -533,7 +533,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 }
 
 func (tab *Table) len() (n int) {
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		n += len(b.entries)
 	}
 	return n

p2p/discv5/net_test.go

@@ -355,7 +355,7 @@ func (tn *preminedTestnet) mine(target NodeID) {
 	fmt.Printf("	target: %#v,\n", tn.target)
 	fmt.Printf("	targetSha: %#v,\n", tn.targetSha)
 	fmt.Printf("	dists: [%d][]NodeID{\n", len(tn.dists))
-	for ld, ns := range tn.dists {
+	for ld, ns := range &tn.dists {
 		if len(ns) == 0 {
 			continue
 		}

p2p/discv5/table.go

@@ -81,7 +81,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
 	if printTable {
 		fmt.Println()
 	}
-	for i, b := range tab.buckets {
+	for i, b := range &tab.buckets {
 		entries += len(b.entries)
 		if printTable {
 			for _, e := range b.entries {
@@ -93,7 +93,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
 	prefix := binary.BigEndian.Uint64(tab.self.sha[0:8])
 	dist := ^uint64(0)
 	entry := int(randUint(uint32(entries + 1)))
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if entry < len(b.entries) {
 			n := b.entries[entry]
 			dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix
@@ -121,7 +121,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) {
 	// TODO: tree-based buckets would help here
 	// Find all non-empty buckets and get a fresh slice of their entries.
 	var buckets [][]*Node
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		if len(b.entries) > 0 {
 			buckets = append(buckets, b.entries[:])
 		}
@@ -175,7 +175,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
 	// obviously correct. I believe that tree-based buckets would make
 	// this easier to implement efficiently.
 	close := &nodesByDistance{target: target}
-	for _, b := range tab.buckets {
+	for _, b := range &tab.buckets {
 		for _, n := range b.entries {
 			close.push(n, nresults)
 		}

swarm/README.md

@@ -7,6 +7,21 @@ Swarm is a distributed storage platform and content distribution service, a nati
 [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
+## Table of Contents
+
+*  [Building the source](#building-the-source)
+*  [Running Swarm](#running-swarm)
+*  [Documentation](#documentation)
+*  [Developers Guide](#developers-guide)
+   *  [Go Environment](#development-environment)
+   *  [Vendored Dependencies](#vendored-dependencies)
+   *  [Testing](#testing)
+   *  [Profiling Swarm](#profiling-swarm)
+   *  [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm)
+*  [Public Gateways](#public-gateways)
+*  [Swarm Dapps](#swarm-dapps)
+*  [Contributing](#contributing)
+*  [License](#license)
 
 ## Building the source
 
@@ -16,13 +31,187 @@ Building Swarm requires Go (version 1.10 or later).
 
     go install github.com/ethereum/go-ethereum/cmd/swarm
 
+## Running Swarm
+
+Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node.
+
+To run Swarm you need an Ethereum account. You can create a new account by running the following command:
+
+    geth account new
+
+You will be prompted for a password:
+
+    Your new account is locked with a password. Please give a password. Do not forget this password.
+    Passphrase:
+    Repeat passphrase:
+
+Once you have specified the password, the output will be the Ethereum address representing that account. For example:
+
+    Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1}
+
+Using this account, connect to Swarm with
+
+    swarm --bzzaccount <your-account-here>
+
+    # in our example
+
+    swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1
+
+
+### Verifying that your local Swarm node is running
+
+When running, Swarm is accessible through an HTTP API on port 8500.
+
+Confirm that it is up and running by pointing your browser to http://localhost:8500
+
+### Ethereum Name Service resolution
+
+The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `geth` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag.
+
+    swarm --bzzaccount <your-account-here> \
+          --ens-api '$HOME/.ethereum/geth.ipc'
+
+    # in our example
+
+    swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \
+          --ens-api '$HOME/.ethereum/geth.ipc'
+
+For more information on usage, features or command line flags, please consult the Documentation.
+
 
 ## Documentation
 
 Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io).
 
 
-## Contribution
+## Developers Guide
+
+### Go Environment
+
+We assume that you have Go v1.10 installed, and `GOPATH` is set.
+
+You must have your working copy under `$GOPATH/src/github.com/ethereum/go-ethereum`.
+
+Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place:
+
+```
+git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ethereum/go-ethereum
+```
+
+
+### Vendored Dependencies
+
+All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them.
+
+If you want to add a new dependency, run `govendor fetch <import-path>`, then commit the result.
+
+If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`.
+
+
+### Testing
+
+This section explains how to run unit, integration, and end-to-end tests in your development sandbox.
+
+Testing one library:
+
+```
+go test -v -cpu 4 ./swarm/api
+```
+
+Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended.
+
+Testing only some methods:
+
+```
+go test -v -cpu 4 ./eth -run TestMethod
+```
+
+Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both!
+
+Running benchmarks:
+
+```
+go test -v -cpu 4 -bench . -run BenchmarkJoin
+```
+
+
+### Profiling Swarm
+
+This section explains how to add Go `pprof` profiler to Swarm
+
+If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060.
+
+You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc.
+
+By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging.
+
+
+### Metrics and Instrumentation in Swarm
+
+This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric.
+
+Swarm metrics system is based on the `go-metrics` library.
+
+The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types.
+
+```
+# incrementing a counter
+metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1)
+
+# measuring latency with a resetting timer
+start := time.Now()
+t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil)
+...
+t := UpdateSince(start)
+```
+
+#### Visualizing metrics
+
+Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it:
+
+```
+swarm --help | grep metrics
+```
+
+We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own.
+
+We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed.
+
+Once you have `stateth` installed, and you have Docker running locally, you have to:
+
+1. Run `stateth` and keep it running in the background
+```
+stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ethereum/go-ethereum/swarm/grafana_dashboards --influxdb-database metrics
+```
+
+2. Run `swarm` with at least the following params:
+```
+--metrics \
+--metrics.influxdb.export \
+--metrics.influxdb.endpoint "http://localhost:8086" \
+--metrics.influxdb.username "admin" \
+--metrics.influxdb.password "admin" \
+--metrics.influxdb.database "metrics"
+```
+
+3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm.
+
+
+## Public Gateways
+
+Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node.
+
+The Swarm public gateways are temporary and users should not rely on their existence for production services.
+
+The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release.
+
+## Swarm Dapps
+
+You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth
+
+Their source code can be found at: https://github.com/ethersphere/swarm-dapps
+
+## Contributing
 
 Thank you for considering to help out with the source code! We welcome contributions from
 anyone on the internet, and are grateful for even the smallest of fixes!

swarm/api/api.go

@@ -339,8 +339,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
 	if err != nil {
 		apiGetNotFound.Inc(1)
 		status = http.StatusNotFound
-		log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
+		return nil, "", http.StatusNotFound, nil, err
-		return
 	}
 
 	log.Debug("trie getting entry", "key", manifestAddr, "path", path)

swarm/api/config.go

@@ -63,6 +63,7 @@ type Config struct {
 	SwapEnabled       bool
 	SyncEnabled       bool
 	DeliverySkipCheck bool
+	LightNodeEnabled  bool
 	SyncUpdateDelay   time.Duration
 	SwapAPI           string
 	Cors              string

swarm/api/http/error.go

@@ -1,208 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-/*
-Show nicely (but simple) formatted HTML error pages (or respond with JSON
-if the appropriate `Accept` header is set)) for the http package.
-*/
-package http
-
-import (
-	"encoding/json"
-	"fmt"
-	"html/template"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/ethereum/go-ethereum/swarm/api"
-	l "github.com/ethereum/go-ethereum/swarm/log"
-)
-
-//templateMap holds a mapping of an HTTP error code to a template
-var templateMap map[int]*template.Template
-var caseErrors []CaseError
-
-//metrics variables
-var (
-	htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
-	jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
-)
-
-//parameters needed for formatting the correct HTML page
-type ResponseParams struct {
-	Msg       string
-	Code      int
-	Timestamp string
-	template  *template.Template
-	Details   template.HTML
-}
-
-//a custom error case struct that would be used to store validators and
-//additional error info to display with client responses.
-type CaseError struct {
-	Validator func(*Request) bool
-	Msg       func(*Request) string
-}
-
-//we init the error handling right on boot time, so lookup and http response is fast
-func init() {
-	initErrHandling()
-}
-
-func initErrHandling() {
-	//pages are saved as strings - get these strings
-	genErrPage := GetGenericErrorPage()
-	notFoundPage := GetNotFoundErrorPage()
-	multipleChoicesPage := GetMultipleChoicesErrorPage()
-	//map the codes to the available pages
-	tnames := map[int]string{
-		0: genErrPage, //default
-		http.StatusBadRequest:          genErrPage,
-		http.StatusNotFound:            notFoundPage,
-		http.StatusMultipleChoices:     multipleChoicesPage,
-		http.StatusInternalServerError: genErrPage,
-	}
-	templateMap = make(map[int]*template.Template)
-	for code, tname := range tnames {
-		//assign formatted HTML to the code
-		templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
-	}
-
-	caseErrors = []CaseError{
-		{
-			Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
-			Msg: func(r *Request) string {
-				uriCopy := r.uri
-				uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
-				return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
-			Please click <a href='%[1]s'>here</a> if your browser does not redirect you.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uriCopy.String())
-			},
-		}}
-}
-
-//ValidateCaseErrors is a method that process the request object through certain validators
-//that assert if certain conditions are met for further information to log as an error
-func ValidateCaseErrors(r *Request) string {
-	for _, err := range caseErrors {
-		if err.Validator(r) {
-			return err.Msg(r)
-		}
-	}
-
-	return ""
-}
-
-//ShowMultipeChoices is used when a user requests a resource in a manifest which results
-//in ambiguous results. It returns a HTML page with clickable links of each of the entry
-//in the manifest which fits the request URI ambiguity.
-//For example, if the user requests bzz:/<hash>/read and that manifest contains entries
-//"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
-//This only applies if the manifest has no default entry
-func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
-	msg := ""
-	if list.Entries == nil {
-		Respond(w, req, "Could not resolve", http.StatusInternalServerError)
-		return
-	}
-	//make links relative
-	//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
-	//to get clickable links, need to remove the ambiguous path, i.e. "read"
-	idx := strings.LastIndex(req.RequestURI, "/")
-	if idx == -1 {
-		Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
-		return
-	}
-	//remove ambiguous part
-	base := req.RequestURI[:idx+1]
-	for _, e := range list.Entries {
-		//create clickable link for each entry
-		msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
-	}
-	Respond(w, req, msg, http.StatusMultipleChoices)
-}
-
-//Respond is used to show an HTML page to a client.
-//If there is an `Accept` header of `application/json`, JSON will be returned instead
-//The function just takes a string message which will be displayed in the error page.
-//The code is used to evaluate which template will be displayed
-//(and return the correct HTTP status code)
-func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
-	additionalMessage := ValidateCaseErrors(req)
-	switch code {
-	case http.StatusInternalServerError:
-		log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
-	case http.StatusMultipleChoices:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
-		listURI := api.URI{
-			Scheme: "bzz-list",
-			Addr:   req.uri.Addr,
-			Path:   req.uri.Path,
-		}
-		additionalMessage = fmt.Sprintf(`<a href="/%s">multiple choices</a>`, listURI.String())
-	default:
-		log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
-	}
-
-	if code >= 400 {
-		w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
-		w.Header().Del("ETag")
-	}
-
-	respond(w, &req.Request, &ResponseParams{
-		Code:      code,
-		Msg:       msg,
-		Details:   template.HTML(additionalMessage),
-		Timestamp: time.Now().Format(time.RFC1123),
-		template:  getTemplate(code),
-	})
-}
-
-//evaluate if client accepts html or json response
-func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
-	w.WriteHeader(params.Code)
-	if r.Header.Get("Accept") == "application/json" {
-		respondJSON(w, params)
-	} else {
-		respondHTML(w, params)
-	}
-}
-
-//return a HTML page
-func respondHTML(w http.ResponseWriter, params *ResponseParams) {
-	htmlCounter.Inc(1)
-	err := params.template.Execute(w, params)
-	if err != nil {
-		log.Error(err.Error())
-	}
-}
-
-//return JSON
-func respondJSON(w http.ResponseWriter, params *ResponseParams) {
-	jsonCounter.Inc(1)
-	w.Header().Set("Content-Type", "application/json")
-	json.NewEncoder(w).Encode(params)
-}
-
-//get the HTML template for a given code
-func getTemplate(code int) *template.Template {
-	if val, tmpl := templateMap[code]; tmpl {
-		return val
-	}
-	return templateMap[0]
-}

swarm/api/http/middleware.go

@@ -0,0 +1,95 @@
+package http
+
+import (
+	"fmt"
+	"net/http"
+	"runtime/debug"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/log"
+	"github.com/ethereum/go-ethereum/swarm/spancontext"
+	"github.com/pborman/uuid"
+)
+
+// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
+// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
+func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
+	for i := range adapters {
+		adapter := adapters[len(adapters)-1-i]
+		h = adapter(h)
+	}
+	return h
+}
+
+type Adapter func(http.Handler) http.Handler
+
+func SetRequestID(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
+		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
+		log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func ParseURI(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
+			return
+		}
+		if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
+			uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
+
+			msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
+			Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
+			w.WriteHeader(http.StatusNotFound)
+			w.Write([]byte(msg))
+			return
+		}
+
+		ctx := r.Context()
+		r = r.WithContext(SetURI(ctx, uri))
+		log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
+
+		h.ServeHTTP(w, r)
+	})
+}
+
+func InitLoggingResponseWriter(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		writer := newLoggingResponseWriter(w)
+
+		h.ServeHTTP(writer, r)
+	})
+}
+
+func InstrumentOpenTracing(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		uri := GetURI(r.Context())
+		if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
+			h.ServeHTTP(w, r) // soft fail
+			return
+		}
+		spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
+		ctx, sp := spancontext.StartSpan(r.Context(), spanName)
+		defer sp.Finish()
+		h.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func RecoverPanic(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer func() {
+			if err := recover(); err != nil {
+				log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
+			}
+		}()
+		h.ServeHTTP(w, r)
+	})
+}

swarm/api/http/response.go

@@ -0,0 +1,132 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/swarm/api"
+)
+
+var (
+	htmlCounter      = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
+	jsonCounter      = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
+	plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
+)
+
+type ResponseParams struct {
+	Msg       template.HTML
+	Code      int
+	Timestamp string
+	template  *template.Template
+	Details   template.HTML
+}
+
+// ShowMultipleChoices is used when a user requests a resource in a manifest which results
+// in ambiguous results. It returns a HTML page with clickable links of each of the entry
+// in the manifest which fits the request URI ambiguity.
+// For example, if the user requests bzz:/<hash>/read and that manifest contains entries
+// "readme.md" and "readinglist.txt", a HTML page is returned with this two links.
+// This only applies if the manifest has no default entry
+func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
+	log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	msg := ""
+	if list.Entries == nil {
+		RespondError(w, r, "Could not resolve", http.StatusInternalServerError)
+		return
+	}
+	requestUri := strings.TrimPrefix(r.RequestURI, "/")
+
+	uri, err := api.Parse(requestUri)
+	if err != nil {
+		RespondError(w, r, "Bad Request", http.StatusBadRequest)
+	}
+
+	uri.Scheme = "bzz-list"
+	msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
+	RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
+}
+
+func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
+	log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	respond(w, r, &ResponseParams{
+		Code:      code,
+		Msg:       template.HTML(msg),
+		Timestamp: time.Now().Format(time.RFC1123),
+		template:  TemplatesMap[templateName],
+	})
+}
+
+func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
+	log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
+	RespondTemplate(w, r, "error", msg, code)
+}
+
+func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+	w.WriteHeader(params.Code)
+
+	if params.Code >= 400 {
+		w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
+		w.Header().Del("ETag")
+	}
+
+	acceptHeader := r.Header.Get("Accept")
+	// this cannot be in a switch form since an Accept header can be in the form of "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
+	if strings.Contains(acceptHeader, "application/json") {
+		if err := respondJSON(w, r, params); err != nil {
+			RespondError(w, r, "Internal server error", http.StatusInternalServerError)
+		}
+	} else if strings.Contains(acceptHeader, "text/html") {
+		respondHTML(w, r, params)
+	} else {
+		respondPlaintext(w, r, params) //returns nice errors for curl
+	}
+}
+
+func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
+	htmlCounter.Inc(1)
+	log.Debug("respondHTML", "ruid", GetRUID(r.Context()))
+	err := params.template.Execute(w, params)
+	if err != nil {
+		log.Error(err.Error())
+	}
+}
+
+func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	jsonCounter.Inc(1)
+	log.Debug("respondJSON", "ruid", GetRUID(r.Context()))
+	w.Header().Set("Content-Type", "application/json")
+	return json.NewEncoder(w).Encode(params)
+}
+
+func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
+	plaintextCounter.Inc(1)
+	log.Debug("respondPlaintext", "ruid", GetRUID(r.Context()))
+	w.Header().Set("Content-Type", "text/plain")
+	strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
+	strToWrite += "Message: " + string(params.Msg) + "\n"
+	strToWrite += "Timestamp: " + params.Timestamp + "\n"
+	_, err := w.Write([]byte(strToWrite))
+	return err
+}

swarm/api/http/response_test.go

@@ -44,7 +44,7 @@ func TestError(t *testing.T) {
 	defer resp.Body.Close()
 	respbody, err = ioutil.ReadAll(resp.Body)
 
-	if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
+	if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI "/this_should_fail_as_no_bzz_protocol_present": unknown scheme") {
 		t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
 	}
 

swarm/api/http/sctx.go

@@ -0,0 +1,38 @@
+package http
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum/swarm/api"
+	"github.com/ethereum/go-ethereum/swarm/sctx"
+)
+
+type contextKey int
+
+const (
+	uriKey contextKey = iota
+)
+
+func GetRUID(ctx context.Context) string {
+	v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string)
+	if ok {
+		return v
+	}
+	return "xxxxxxxx"
+}
+
+func SetRUID(ctx context.Context, ruid string) context.Context {
+	return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid)
+}
+
+func GetURI(ctx context.Context) *api.URI {
+	v, ok := ctx.Value(uriKey).(*api.URI)
+	if ok {
+		return v
+	}
+	return nil
+}
+
+func SetURI(ctx context.Context, uri *api.URI) context.Context {
+	return context.WithValue(ctx, uriKey, uri)
+}

swarm/api/http/server.go

@@ -41,12 +41,9 @@ import (
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/swarm/api"
 	"github.com/ethereum/go-ethereum/swarm/log"
-	"github.com/ethereum/go-ethereum/swarm/spancontext"
 	"github.com/ethereum/go-ethereum/swarm/storage"
 	"github.com/ethereum/go-ethereum/swarm/storage/mru"
-	opentracing "github.com/opentracing/opentracing-go"
 
-	"github.com/pborman/uuid"
 	"github.com/rs/cors"
 )
 
@@ -72,6 +69,17 @@ var (
 	getListFail     = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
 )
 
+type methodHandler map[string]http.Handler
+
+func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
+	v, ok := m[r.Method]
+	if ok {
+		v.ServeHTTP(rw, r)
+		return
+	}
+	rw.WriteHeader(http.StatusMethodNotAllowed)
+}
+
 func NewServer(api *api.API, corsString string) *Server {
 	var allowedOrigins []string
 	for _, domain := range strings.Split(corsString, ",") {
@@ -84,20 +92,79 @@ func NewServer(api *api.API, corsString string) *Server {
 		AllowedHeaders: []string{"*"},
 	})
 
-	mux := http.NewServeMux()
 	server := &Server{api: api}
-	mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz))
-	mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw))
-	mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable))
-	mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash))
-	mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList))
-	mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource))
 
-	mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths))
+	defaultMiddlewares := []Adapter{
-	mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths))
+		RecoverPanic,
-	mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths))
+		SetRequestID,
+		InitLoggingResponseWriter,
+		ParseURI,
+		InstrumentOpenTracing,
+	}
 
+	mux := http.NewServeMux()
+	mux.Handle("/bzz:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleBzzGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostFiles),
+			defaultMiddlewares...,
+		),
+		"DELETE": Adapt(
+			http.HandlerFunc(server.HandleDelete),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-raw:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostRaw),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-immutable:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-hash:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGet),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-list:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetList),
+			defaultMiddlewares...,
+		),
+	})
+	mux.Handle("/bzz-resource:/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleGetResource),
+			defaultMiddlewares...,
+		),
+		"POST": Adapt(
+			http.HandlerFunc(server.HandlePostResource),
+			defaultMiddlewares...,
+		),
+	})
+
+	mux.Handle("/", methodHandler{
+		"GET": Adapt(
+			http.HandlerFunc(server.HandleRootPaths),
+			SetRequestID,
+			InitLoggingResponseWriter,
+		),
+	})
 	server.Handler = c.Handler(mux)
+
 	return server
 }
 
@@ -105,139 +172,6 @@ func (s *Server) ListenAndServe(addr string) error {
 	return http.ListenAndServe(addr, s)
 }
 
-func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		if r.RequestURI == "/" {
-			if strings.Contains(r.Header.Get("Accept"), "text/html") {
-				err := landingPageTemplate.Execute(w, nil)
-				if err != nil {
-					log.Error(fmt.Sprintf("error rendering landing page: %s", err))
-				}
-				return
-			}
-			if strings.Contains(r.Header.Get("Accept"), "application/json") {
-				w.Header().Set("Content-Type", "application/json")
-				w.WriteHeader(http.StatusOK)
-				json.NewEncoder(w).Encode("Welcome to Swarm!")
-				return
-			}
-		}
-
-		if r.URL.Path == "/robots.txt" {
-			w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
-			fmt.Fprintf(w, "User-agent: *\nDisallow: /")
-			return
-		}
-		Respond(w, r, "Bad Request", http.StatusBadRequest)
-	default:
-		Respond(w, r, "Not Found", http.StatusNotFound)
-	}
-}
-
-func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetBzz")
-		if r.Header.Get("Accept") == "application/x-tar" {
-			reader, err := s.api.GetDirectoryTar(r.Context(), r.uri)
-			if err != nil {
-				Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
-			}
-			defer reader.Close()
-
-			w.Header().Set("Content-Type", "application/x-tar")
-			w.WriteHeader(http.StatusOK)
-			io.Copy(w, reader)
-			return
-		}
-		s.HandleGetFile(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostFiles")
-		s.HandlePostFiles(w, r)
-	case http.MethodDelete:
-		log.Debug("handleBzzDelete")
-		s.HandleDelete(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetRaw")
-		s.HandleGet(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostRaw")
-		s.HandlePostRaw(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGetList(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGet(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetHash")
-		s.HandleGetList(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) {
-	switch r.Method {
-	case http.MethodGet:
-		log.Debug("handleGetResource")
-		s.HandleGetResource(w, r)
-	case http.MethodPost:
-		log.Debug("handlePostResource")
-		s.HandlePostResource(w, r)
-	default:
-		Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
-	}
-}
-func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc {
-	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-		defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
-		req := &Request{Request: *r, ruid: uuid.New()[:8]}
-		metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
-		log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
-
-		// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
-		w := newLoggingResponseWriter(rw)
-		if parseBzzUri {
-			uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
-			if err != nil {
-				Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
-				return
-			}
-			req.uri = uri
-
-			log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
-		}
-
-		h(w, req) // call original
-		log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
-	})
-}
-
 // browser API for registering bzz url scheme handlers:
 // https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
 // electron (chromium) api for registering bzz url scheme handlers:
@@ -247,59 +181,81 @@ type Server struct {
 	api *api.API
 }
 
-// Request wraps http.Request and also includes the parsed bzz URI
+func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
-type Request struct {
+	log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()))
-	http.Request
+	if r.Header.Get("Accept") == "application/x-tar" {
+		uri := GetURI(r.Context())
+		reader, err := s.api.GetDirectoryTar(r.Context(), uri)
+		if err != nil {
+			RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
+		}
+		defer reader.Close()
 
-	uri  *api.URI
+		w.Header().Set("Content-Type", "application/x-tar")
-	ruid string // request unique id
+		w.WriteHeader(http.StatusOK)
+		io.Copy(w, reader)
+		return
+	}
+
+	s.HandleGetFile(w, r)
+}
+
+func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) {
+	switch r.RequestURI {
+	case "/":
+		RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200)
+		return
+	case "/robots.txt":
+		w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
+		fmt.Fprintf(w, "User-agent: *\nDisallow: /")
+	case "/favicon.ico":
+		w.WriteHeader(http.StatusOK)
+		w.Write(faviconBytes)
+	default:
+		RespondError(w, r, "Not Found", http.StatusNotFound)
+	}
 }
 
 // HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
 // body in swarm and returns the resulting storage address as a text/plain response
-func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.raw", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.raw", "ruid", ruid)
 
 	postRawCount.Inc(1)
 
-	ctx := r.Context()
-	var sp opentracing.Span
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.raw")
-	defer sp.Finish()
-
 	toEncrypt := false
-	if r.uri.Addr == "encrypt" {
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
 		toEncrypt = true
 	}
 
-	if r.uri.Path != "" {
+	if uri.Path != "" {
 		postRawFail.Inc(1)
-		Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
+		RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
 		return
 	}
 
-	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != "encrypt" {
 		postRawFail.Inc(1)
-		Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
+		RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
 		return
 	}
 
 	if r.Header.Get("Content-Length") == "" {
 		postRawFail.Inc(1)
-		Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
+		RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest)
 		return
 	}
 
-	addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt)
+	addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
 	if err != nil {
 		postRawFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	log.Debug("stored content", "ruid", r.ruid, "key", addr)
+	log.Debug("stored content", "ruid", ruid, "key", addr)
 
 	w.Header().Set("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
@@ -311,55 +267,49 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
 // (either a tar archive or multipart form), adds those files either to an
 // existing manifest or to a new manifest under <path> and returns the
 // resulting manifest hash as a text/plain response
-func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.files", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	log.Debug("handle.post.files", "ruid", ruid)
 	postFilesCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.files")
-	defer sp.Finish()
-
 	contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
 	if err != nil {
 		postFilesFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusBadRequest)
+		RespondError(w, r, err.Error(), http.StatusBadRequest)
 		return
 	}
 
 	toEncrypt := false
-	if r.uri.Addr == "encrypt" {
+	uri := GetURI(r.Context())
+	if uri.Addr == "encrypt" {
 		toEncrypt = true
 	}
 
 	var addr storage.Address
-	if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
+	if uri.Addr != "" && uri.Addr != "encrypt" {
-		addr, err = s.api.Resolve(r.Context(), r.uri)
+		addr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			postFilesFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
 			return
 		}
-		log.Debug("resolved key", "ruid", r.ruid, "key", addr)
+		log.Debug("resolved key", "ruid", ruid, "key", addr)
 	} else {
 		addr, err = s.api.NewManifest(r.Context(), toEncrypt)
 		if err != nil {
 			postFilesFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
-		log.Debug("new manifest", "ruid", r.ruid, "key", addr)
+		log.Debug("new manifest", "ruid", ruid, "key", addr)
 	}
 
-	newAddr, err := s.api.UpdateManifest(ctx, addr, func(mw *api.ManifestWriter) error {
+	newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
 		switch contentType {
-
 		case "application/x-tar":
 			_, err := s.handleTarUpload(r, mw)
 			if err != nil {
-				Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
+				RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
 				return err
 			}
 			return nil
@@ -372,30 +322,31 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
 	})
 	if err != nil {
 		postFilesFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
+		RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
 		return
 	}
 
-	log.Debug("stored content", "ruid", r.ruid, "key", newAddr)
+	log.Debug("stored content", "ruid", ruid, "key", newAddr)
 
 	w.Header().Set("Content-Type", "text/plain")
 	w.WriteHeader(http.StatusOK)
 	fmt.Fprint(w, newAddr)
 }
 
-func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) {
+func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
-	log.Debug("handle.tar.upload", "ruid", r.ruid)
+	log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()))
 
-	key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw)
+	key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, mw)
 	if err != nil {
 		return nil, err
 	}
 	return key, nil
 }
 
-func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
+func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error {
-	log.Debug("handle.multipart.upload", "ruid", req.ruid)
+	ruid := GetRUID(r.Context())
-	mr := multipart.NewReader(req.Body, boundary)
+	log.Debug("handle.multipart.upload", "ruid", ruid)
+	mr := multipart.NewReader(r.Body, boundary)
 	for {
 		part, err := mr.NextPart()
 		if err == io.EOF {
@@ -435,48 +386,52 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
 		if name == "" {
 			name = part.FormName()
 		}
-		path := path.Join(req.uri.Path, name)
+		uri := GetURI(r.Context())
+		path := path.Join(uri.Path, name)
 		entry := &api.ManifestEntry{
 			Path:        path,
 			ContentType: part.Header.Get("Content-Type"),
 			Size:        size,
 			ModTime:     time.Now(),
 		}
-		log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
+		log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path)
-		contentKey, err := mw.AddEntry(req.Context(), reader, entry)
+		contentKey, err := mw.AddEntry(r.Context(), reader, entry)
 		if err != nil {
 			return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
 		}
-		log.Debug("stored content", "ruid", req.ruid, "key", contentKey)
+		log.Debug("stored content", "ruid", ruid, "key", contentKey)
 	}
 }
 
-func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
+func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error {
-	log.Debug("handle.direct.upload", "ruid", req.ruid)
+	ruid := GetRUID(r.Context())
-	key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{
+	log.Debug("handle.direct.upload", "ruid", ruid)
-		Path:        req.uri.Path,
+	key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{
-		ContentType: req.Header.Get("Content-Type"),
+		Path:        GetURI(r.Context()).Path,
+		ContentType: r.Header.Get("Content-Type"),
 		Mode:        0644,
-		Size:        req.ContentLength,
+		Size:        r.ContentLength,
 		ModTime:     time.Now(),
 	})
 	if err != nil {
 		return err
 	}
-	log.Debug("stored content", "ruid", req.ruid, "key", key)
+	log.Debug("stored content", "ruid", ruid, "key", key)
 	return nil
 }
 
 // HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
 // <path> from <manifest> and returns the resulting manifest hash as a
 // text/plain response
-func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.delete", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.delete", "ruid", ruid)
 	deleteCount.Inc(1)
-	newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path)
+	newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path)
 	if err != nil {
 		deleteFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
+		RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
 		return
 	}
 
@@ -519,27 +474,20 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) {
 //
 // The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON`
 // The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content
-func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
+func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.post.resource", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
-
+	log.Debug("handle.post.resource", "ruid", ruid)
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.post.resource")
-	defer sp.Finish()
-
 	var err error
 
 	// Creation and update must send mru.updateRequestJSON JSON structure
 	body, err := ioutil.ReadAll(r.Body)
 	if err != nil {
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	var updateRequest mru.Request
 	if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON
-		Respond(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
+		RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
 		return
 	}
 
@@ -548,7 +496,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// to update this resource
 		// Check this early, to avoid creating a resource and then not being able to set its first update.
 		if err = updateRequest.Verify(); err != nil {
-			Respond(w, r, err.Error(), http.StatusForbidden)
+			RespondError(w, r, err.Error(), http.StatusForbidden)
 			return
 		}
 	}
@@ -557,7 +505,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		err = s.api.ResourceCreate(r.Context(), &updateRequest)
 		if err != nil {
 			code, err2 := s.translateResourceError(w, r, "resource creation fail", err)
-			Respond(w, r, err2.Error(), code)
+			RespondError(w, r, err2.Error(), code)
 			return
 		}
 	}
@@ -565,7 +513,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 	if updateRequest.IsUpdate() {
 		_, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate)
 		if err != nil {
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
 	}
@@ -579,7 +527,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// metadata chunk (rootAddr)
 		m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex())
 		if err != nil {
-			Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
 			return
 		}
 
@@ -589,7 +537,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 		// \TODO update manifest key automatically in ENS
 		outdata, err := json.Marshal(m)
 		if err != nil {
-			Respond(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
+			RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
 			return
 		}
 		fmt.Fprint(w, string(outdata))
@@ -604,17 +552,19 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
 // bzz-resource://<id>/meta - get metadata and next version information
 // <id> = ens name or hash
 // TODO: Enable pass maxPeriod parameter
-func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.resource", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.resource", "ruid", ruid)
 	var err error
 
 	// resolve the content key.
-	manifestAddr := r.uri.Address()
+	manifestAddr := uri.Address()
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
@@ -625,25 +575,25 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
 	if err != nil {
 		getFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound)
 		return
 	}
 
-	log.Debug("handle.get.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
+	log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
 
 	// determine if the query specifies period and version or it is a metadata query
 	var params []string
-	if len(r.uri.Path) > 0 {
+	if len(uri.Path) > 0 {
-		if r.uri.Path == "meta" {
+		if uri.Path == "meta" {
 			unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr)
 			if err != nil {
 				getFail.Inc(1)
-				Respond(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
+				RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
 				return
 			}
 			rawResponse, err := unsignedUpdateRequest.MarshalJSON()
 			if err != nil {
-				Respond(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
+				RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
 				return
 			}
 			w.Header().Add("Content-type", "application/json")
@@ -653,7 +603,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 
 		}
 
-		params = strings.Split(r.uri.Path, "/")
+		params = strings.Split(uri.Path, "/")
 
 	}
 	var name string
@@ -689,17 +639,17 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
 	// any error from the switch statement will end up here
 	if err != nil {
 		code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err)
-		Respond(w, r, err2.Error(), code)
+		RespondError(w, r, err2.Error(), code)
 		return
 	}
 
 	// All ok, serve the retrieved update
-	log.Debug("Found update", "name", name, "ruid", r.ruid)
+	log.Debug("Found update", "name", name, "ruid", ruid)
 	w.Header().Set("Content-Type", "application/octet-stream")
-	http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data))
+	http.ServeContent(w, r, "", now, bytes.NewReader(data))
 }
 
-func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) (int, error) {
+func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {
 	code := 0
 	defaultErr := fmt.Errorf("%s: %v", supErr, err)
 	rsrcErr, ok := err.(*mru.Error)
@@ -725,46 +675,41 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
 //   given storage key
 // - bzz-hash://<key> and responds with the hash of the content stored
 //   at the given storage key as a text/plain response
-func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get", "ruid", ruid, "uri", uri)
 	getCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get")
-	defer sp.Finish()
-
 	var err error
-	addr := r.uri.Address()
+	addr := uri.Address()
 	if addr == nil {
-		addr, err = s.api.Resolve(r.Context(), r.uri)
+		addr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
 		w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
 
-	log.Debug("handle.get: resolved", "ruid", r.ruid, "key", addr)
+	log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
 
 	// if path is set, interpret <key> as a manifest and return the
 	// raw entry at the given path
-	if r.uri.Path != "" {
+	if uri.Path != "" {
 		walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
 		if err != nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
+			RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
 			return
 		}
 		var entry *api.ManifestEntry
 		walker.Walk(func(e *api.ManifestEntry) error {
 			// if the entry matches the path, set entry and stop
 			// the walk
-			if e.Path == r.uri.Path {
+			if e.Path == uri.Path {
 				entry = e
 				// return an error to cancel the walk
 				return errors.New("found")
@@ -778,7 +723,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 			// if the manifest's path is a prefix of the
 			// requested path, recurse into it by returning
 			// nil and continuing the walk
-			if strings.HasPrefix(r.uri.Path, e.Path) {
+			if strings.HasPrefix(uri.Path, e.Path) {
 				return nil
 			}
 
@@ -786,7 +731,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 		})
 		if entry == nil {
 			getFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
 			return
 		}
 		addr = storage.Address(common.Hex2Bytes(entry.Hash))
@@ -796,23 +741,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
 	if noneMatchEtag != "" {
 		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) {
-			Respond(w, r, "Not Modified", http.StatusNotModified)
+			w.WriteHeader(http.StatusNotModified)
 			return
 		}
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	reader, isEncrypted := s.api.Retrieve(ctx, addr)
+	reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
-	if _, err := reader.Size(ctx, nil); err != nil {
+	if _, err := reader.Size(r.Context(), nil); err != nil {
 		getFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
 		return
 	}
 
 	w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
 
 	switch {
-	case r.uri.Raw():
+	case uri.Raw():
 		// allow the request to overwrite the content type using a query
 		// parameter
 		contentType := "application/octet-stream"
@@ -820,8 +765,8 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 			contentType = typ
 		}
 		w.Header().Set("Content-Type", contentType)
-		http.ServeContent(w, &r.Request, "", time.Now(), reader)
+		http.ServeContent(w, r, "", time.Now(), reader)
-	case r.uri.Hash():
+	case uri.Hash():
 		w.Header().Set("Content-Type", "text/plain")
 		w.WriteHeader(http.StatusOK)
 		fmt.Fprint(w, addr)
@@ -831,35 +776,30 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
 // HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
 // a list of all files contained in <manifest> under <path> grouped into
 // common prefixes using "/" as a delimiter
-func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
 	getListCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get.list")
-	defer sp.Finish()
-
 	// ensure the root path has a trailing slash so that relative URLs work
-	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 
-	addr, err := s.api.Resolve(r.Context(), r.uri)
+	addr, err := s.api.Resolve(r.Context(), uri)
 	if err != nil {
 		getListFail.Inc(1)
-		Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 		return
 	}
-	log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr)
+	log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
 
-	list, err := s.api.GetManifestList(ctx, addr, r.uri.Path)
+	list, err := s.api.GetManifestList(r.Context(), addr, uri.Path)
 	if err != nil {
 		getListFail.Inc(1)
-		Respond(w, r, err.Error(), http.StatusInternalServerError)
+		RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
@@ -867,11 +807,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 	// HTML index with relative URLs
 	if strings.Contains(r.Header.Get("Accept"), "text/html") {
 		w.Header().Set("Content-Type", "text/html")
-		err := htmlListTemplate.Execute(w, &htmlListData{
+		err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{
 			URI: &api.URI{
 				Scheme: "bzz",
-				Addr:   r.uri.Addr,
+				Addr:   uri.Addr,
-				Path:   r.uri.Path,
+				Path:   uri.Path,
 			},
 			List: &list,
 		})
@@ -888,45 +828,40 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
 
 // HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
 // with the content of the file at <path> from the given <manifest>
-func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
+func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
-	log.Debug("handle.get.file", "ruid", r.ruid)
+	ruid := GetRUID(r.Context())
+	uri := GetURI(r.Context())
+	log.Debug("handle.get.file", "ruid", ruid)
 	getFileCount.Inc(1)
 
-	var sp opentracing.Span
-	ctx := r.Context()
-	ctx, sp = spancontext.StartSpan(
-		ctx,
-		"http.get.file")
-	defer sp.Finish()
-
 	// ensure the root path has a trailing slash so that relative URLs work
-	if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
+	if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
-		http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
+		http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
 		return
 	}
 	var err error
-	manifestAddr := r.uri.Address()
+	manifestAddr := uri.Address()
 
 	if manifestAddr == nil {
-		manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
+		manifestAddr, err = s.api.Resolve(r.Context(), uri)
 		if err != nil {
 			getFileFail.Inc(1)
-			Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
+			RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
 			return
 		}
 	} else {
 		w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
 	}
 
-	log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr)
+	log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
-	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path)
+	reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path)
 
 	etag := common.Bytes2Hex(contentKey)
 	noneMatchEtag := r.Header.Get("If-None-Match")
 	w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key.
 	if noneMatchEtag != "" {
 		if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) {
-			Respond(w, r, "Not Modified", http.StatusNotModified)
+			w.WriteHeader(http.StatusNotModified)
 			return
 		}
 	}
@@ -935,10 +870,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 		switch status {
 		case http.StatusNotFound:
 			getFileNotFound.Inc(1)
-			Respond(w, r, err.Error(), http.StatusNotFound)
+			RespondError(w, r, err.Error(), http.StatusNotFound)
 		default:
 			getFileFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 		}
 		return
 	}
@@ -946,28 +881,28 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
 	//the request results in ambiguous files
 	//e.g. /read with readme.md and readinglist.txt available in manifest
 	if status == http.StatusMultipleChoices {
-		list, err := s.api.GetManifestList(ctx, manifestAddr, r.uri.Path)
+		list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path)
 		if err != nil {
 			getFileFail.Inc(1)
-			Respond(w, r, err.Error(), http.StatusInternalServerError)
+			RespondError(w, r, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
-		log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", r.ruid)
+		log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid)
 		//show a nice page links to available entries
 		ShowMultipleChoices(w, r, list)
 		return
 	}
 
 	// check the root chunk exists by retrieving the file's size
-	if _, err := reader.Size(ctx, nil); err != nil {
+	if _, err := reader.Size(r.Context(), nil); err != nil {
 		getFileNotFound.Inc(1)
-		Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound)
+		RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound)
 		return
 	}
 
 	w.Header().Set("Content-Type", contentType)
-	http.ServeContent(w, &r.Request, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
+	http.ServeContent(w, r, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
 }
 
 // The size of buffer used for bufio.Reader on LazyChunkReader passed to

swarm/api/manifest.go

@@ -159,7 +159,7 @@ func (m *ManifestWalker) Walk(walkFn WalkFn) error {
 }
 
 func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
-	for _, entry := range trie.entries {
+	for _, entry := range &trie.entries {
 		if entry == nil {
 			continue
 		}
@@ -308,7 +308,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
 }
 
 func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
-	for _, e := range mt.entries {
+	for _, e := range &mt.entries {
 		if e != nil {
 			cnt++
 			entry = e
@@ -362,7 +362,7 @@ func (mt *manifestTrie) recalcAndStore() error {
 	buffer.WriteString(`{"entries":[`)
 
 	list := &Manifest{}
-	for _, entry := range mt.entries {
+	for _, entry := range &mt.entries {
 		if entry != nil {
 			if entry.Hash == "" { // TODO: paralellize
 				err := entry.subtrie.recalcAndStore()

swarm/bmt/bmt.go

@@ -318,7 +318,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) {
 // with every full segment calls writeSection in a go routine
 func (h *Hasher) Write(b []byte) (int, error) {
 	l := len(b)
-	if l == 0 {
+	if l == 0 || l > 4096 {
 		return 0, nil
 	}
 	t := h.getTree()

swarm/network/protocol.go

@@ -94,12 +94,14 @@ type BzzConfig struct {
 	UnderlayAddr []byte // node's underlay address
 	HiveParams   *HiveParams
 	NetworkID    uint64
+	LightNode    bool
 }
 
 // Bzz is the swarm protocol bundle
 type Bzz struct {
 	*Hive
 	NetworkID    uint64
+	LightNode    bool
 	localAddr    *BzzAddr
 	mtx          sync.Mutex
 	handshakes   map[discover.NodeID]*HandshakeMsg
@@ -116,6 +118,7 @@ func NewBzz(config *BzzConfig, kad Overlay, store state.Store, streamerSpec *pro
 	return &Bzz{
 		Hive:         NewHive(config.HiveParams, kad, store),
 		NetworkID:    config.NetworkID,
+		LightNode:    config.LightNode,
 		localAddr:    &BzzAddr{config.OverlayAddr, config.UnderlayAddr},
 		handshakes:   make(map[discover.NodeID]*HandshakeMsg),
 		streamerRun:  streamerRun,
@@ -209,7 +212,11 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*
 			localAddr:  b.localAddr,
 			BzzAddr:    handshake.peerAddr,
 			lastActive: time.Now(),
+			LightNode:  handshake.LightNode,
 		}
+
+		log.Debug("peer created", "addr", handshake.peerAddr.String())
+
 		return run(peer)
 	}
 }
@@ -228,6 +235,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error
 		return err
 	}
 	handshake.peerAddr = rsh.(*HandshakeMsg).Addr
+	handshake.LightNode = rsh.(*HandshakeMsg).LightNode
 	return nil
 }
 
@@ -263,6 +271,7 @@ type BzzPeer struct {
 	localAddr       *BzzAddr  // local Peers address
 	*BzzAddr                  // remote address -> implements Addr interface = protocols.Peer
 	lastActive      time.Time // time is updated whenever mutexes are releasing
+	LightNode       bool
 }
 
 func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer {
@@ -294,6 +303,7 @@ type HandshakeMsg struct {
 	Version   uint64
 	NetworkID uint64
 	Addr      *BzzAddr
+	LightNode bool
 
 	// peerAddr is the address received in the peer handshake
 	peerAddr *BzzAddr
@@ -305,7 +315,7 @@ type HandshakeMsg struct {
 
 // String pretty prints the handshake
 func (bh *HandshakeMsg) String() string {
-	return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", bh.Version, bh.NetworkID, bh.Addr)
+	return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, LightNode: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.LightNode, bh.peerAddr)
 }
 
 // Perform initiates the handshake and validates the remote handshake message
@@ -338,6 +348,7 @@ func (b *Bzz) GetHandshake(peerID discover.NodeID) (*HandshakeMsg, bool) {
 			Version:   uint64(BzzSpec.Version),
 			NetworkID: b.NetworkID,
 			Addr:      b.localAddr,
+			LightNode: b.LightNode,
 			init:      make(chan bool, 1),
 			done:      make(chan struct{}),
 		}

swarm/network/protocol_test.go

@@ -30,6 +30,11 @@ import (
 	p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
 )
 
+const (
+	TestProtocolVersion   = 5
+	TestProtocolNetworkID = 3
+)
+
 var (
 	loglevel = flag.Int("loglevel", 2, "verbosity of logs")
 )
@@ -127,23 +132,30 @@ type bzzTester struct {
 	*p2ptest.ProtocolTester
 	addr *BzzAddr
 	cs   map[string]chan bool
+	bzz  *Bzz
 }
 
-func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr) *bzzTester {
+func newBzz(addr *BzzAddr, lightNode bool) *Bzz {
 	config := &BzzConfig{
 		OverlayAddr:  addr.Over(),
 		UnderlayAddr: addr.Under(),
 		HiveParams:   NewHiveParams(),
 		NetworkID:    DefaultNetworkID,
+		LightNode:    lightNode,
 	}
 	kad := NewKademlia(addr.OAddr, NewKadParams())
 	bzz := NewBzz(config, kad, nil, nil, nil)
+	return bzz
+}
 
-	s := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, bzz.runBzz)
+func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *bzzTester {
+	bzz := newBzz(addr, lightNode)
+	pt := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), n, bzz.runBzz)
 
 	return &bzzTester{
 		addr:           addr,
-		ProtocolTester: s,
+		ProtocolTester: pt,
+		bzz:            bzz,
 	}
 }
 
@@ -184,22 +196,24 @@ func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptes
 	return nil
 }
 
-func correctBzzHandshake(addr *BzzAddr) *HandshakeMsg {
+func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg {
 	return &HandshakeMsg{
-		Version:   5,
+		Version:   TestProtocolVersion,
-		NetworkID: DefaultNetworkID,
+		NetworkID: TestProtocolNetworkID,
 		Addr:      addr,
+		LightNode: lightNode,
 	}
 }
 
 func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 5, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: TestProtocolVersion, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
 		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): network id mismatch 321 (!= 3)")},
 	)
 
@@ -209,14 +223,15 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
 }
 
 func TestBzzHandshakeVersionMismatch(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 0, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: 0, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
-		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= 5)")},
+		&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= %d)", TestProtocolVersion)},
 	)
 
 	if err != nil {
@@ -225,16 +240,49 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) {
 }
 
 func TestBzzHandshakeSuccess(t *testing.T) {
+	lightNode := false
 	addr := RandomAddr()
-	s := newBzzHandshakeTester(t, 1, addr)
+	s := newBzzHandshakeTester(t, 1, addr, lightNode)
 	id := s.IDs[0]
 
 	err := s.testHandshake(
-		correctBzzHandshake(addr),
+		correctBzzHandshake(addr, lightNode),
-		&HandshakeMsg{Version: 5, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
+		&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
 	)
 
 	if err != nil {
 		t.Fatal(err)
 	}
 }
+
+func TestBzzHandshakeLightNode(t *testing.T) {
+	var lightNodeTests = []struct {
+		name      string
+		lightNode bool
+	}{
+		{"on", true},
+		{"off", false},
+	}
+
+	for _, test := range lightNodeTests {
+		t.Run(test.name, func(t *testing.T) {
+			randomAddr := RandomAddr()
+			pt := newBzzHandshakeTester(t, 1, randomAddr, false)
+			id := pt.IDs[0]
+			addr := NewAddrFromNodeID(id)
+
+			err := pt.testHandshake(
+				correctBzzHandshake(randomAddr, false),
+				&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: addr, LightNode: test.lightNode},
+			)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if pt.bzz.handshakes[id].LightNode != test.lightNode {
+				t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[id].LightNode, test.lightNode)
+			}
+		})
+	}
+}

swarm/sctx/sctx.go

@@ -0,0 +1,7 @@
+package sctx
+
+type ContextKey int
+
+const (
+	HTTPRequestIDKey ContextKey = iota
+)

swarm/storage/mru/handler.go

@@ -469,7 +469,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd
 	log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.SData, "multihash", r.multihash)
 
 	// update our resources map entry if the new update is older than the one we have, if we have it.
-	if rsrc != nil && r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version) {
+	if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) {
 		rsrc.period = r.period
 		rsrc.version = r.version
 		rsrc.data = make([]byte, len(r.data))

swarm/swarm.go

@@ -143,6 +143,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
 		OverlayAddr:  addr.OAddr,
 		UnderlayAddr: addr.UAddr,
 		HiveParams:   config.HiveParams,
+		LightNode:    config.LightNodeEnabled,
 	}
 
 	stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db"))

trie/database.go

@@ -51,7 +51,7 @@ const secureKeyLength = 11 + 32
 
 // DatabaseReader wraps the Get and Has method of a backing store for the trie.
 type DatabaseReader interface {
-	// Get retrieves the value associated with key form the database.
+	// Get retrieves the value associated with key from the database.
 	Get(key []byte) (value []byte, err error)
 
 	// Has retrieves whether a key is present in the database.

trie/node.go

@@ -55,7 +55,7 @@ var nilValueNode = valueNode(nil)
 func (n *fullNode) EncodeRLP(w io.Writer) error {
 	var nodes [17]node
 
-	for i, child := range n.Children {
+	for i, child := range &n.Children {
 		if child != nil {
 			nodes[i] = child
 		} else {
@@ -98,7 +98,7 @@ func (n valueNode) String() string  { return n.fstring("") }
 
 func (n *fullNode) fstring(ind string) string {
 	resp := fmt.Sprintf("[\n%s  ", ind)
-	for i, node := range n.Children {
+	for i, node := range &n.Children {
 		if node == nil {
 			resp += fmt.Sprintf("%s: <nil> ", indices[i])
 		} else {

trie/trie.go

@@ -356,7 +356,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		// value that is left in n or -2 if n contains at least two
 		// values.
 		pos := -1
-		for i, cld := range n.Children {
+		for i, cld := range &n.Children {
 			if cld != nil {
 				if pos == -1 {
 					pos = i

whisper/whisperv6/doc.go

@@ -33,7 +33,6 @@ particularly the notion of singular endpoints.
 package whisperv6
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -79,12 +78,6 @@ const (
 	DefaultSyncAllowance = 10 // seconds
 )
 
-type unknownVersionError uint64
-
-func (e unknownVersionError) Error() string {
-	return fmt.Sprintf("invalid envelope version %d", uint64(e))
-}
-
 // MailServer represents a mail server, capable of
 // archiving the old messages for subsequent delivery
 // to the peers. Any implementation must ensure that both