forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-19 10:22:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #15 from ethereum/master

Browse source 
Pull from go-ethereum and merge
This commit is contained in:
Caesar Chad 2018-08-09 10:00:09 +08:00 committed by GitHub
commit 3d9f720de2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 1373 additions and 1312 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cmd/swarm/config.go
View file

@ -68,6 +68,7 @@ const (
SWARM_ENV_SWAP_API = "SWARM_SWAP_API"
SWARM_ENV_SYNC_DISABLE = "SWARM_SYNC_DISABLE"
SWARM_ENV_SYNC_UPDATE_DELAY = "SWARM_ENV_SYNC_UPDATE_DELAY"
SWARM_ENV_LIGHT_NODE_ENABLE = "SWARM_LIGHT_NODE_ENABLE"
SWARM_ENV_DELIVERY_SKIP_CHECK = "SWARM_DELIVERY_SKIP_CHECK"
SWARM_ENV_ENS_API = "SWARM_ENS_API"
SWARM_ENV_ENS_ADDR = "SWARM_ENS_ADDR"
@ -204,6 +205,10 @@ func cmdLineOverride(currentConfig *bzzapi.Config, ctx *cli.Context) *bzzapi.Con
currentConfig.SyncUpdateDelay = d
}
if ctx.GlobalIsSet(SwarmLightNodeEnabled.Name) {
currentConfig.LightNodeEnabled = true
}
if ctx.GlobalIsSet(SwarmDeliverySkipCheckFlag.Name) {
currentConfig.DeliverySkipCheck = true
}
@ -301,6 +306,12 @@ func envVarsOverride(currentConfig *bzzapi.Config) (config *bzzapi.Config) {
}
}
if lne := os.Getenv(SWARM_ENV_LIGHT_NODE_ENABLE); lne != "" {
if lightnode, err := strconv.ParseBool(lne); err != nil {
currentConfig.LightNodeEnabled = lightnode
}
}
if swapapi := os.Getenv(SWARM_ENV_SWAP_API); swapapi != "" {
currentConfig.SwapAPI = swapapi
}

6
cmd/swarm/main.go
View file

@ -123,6 +123,11 @@ var (
Usage: "Duration for sync subscriptions update after no new peers are added (default 15s)",
EnvVar: SWARM_ENV_SYNC_UPDATE_DELAY,
}
SwarmLightNodeEnabled = cli.BoolFlag{
Name: "lightnode",
Usage: "Enable Swarm LightNode (default false)",
EnvVar: SWARM_ENV_LIGHT_NODE_ENABLE,
}
SwarmDeliverySkipCheckFlag = cli.BoolFlag{
Name: "delivery-skip-check",
Usage: "Skip chunk delivery check (default false)",
@ -464,6 +469,7 @@ pv(1) tool to get a progress bar:
SwarmSwapAPIFlag,
SwarmSyncDisabledFlag,
SwarmSyncUpdateDelay,
SwarmLightNodeEnabled,
SwarmDeliverySkipCheckFlag,
SwarmListenAddrFlag,
SwarmPortFlag,

8
p2p/discover/table.go
View file

@ -160,7 +160,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) {
// Find all non-empty buckets and get a fresh slice of their entries.
var buckets [][]*Node
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
if len(b.entries) > 0 {
buckets = append(buckets, b.entries[:])
}
@ -508,7 +508,7 @@ func (tab *Table) copyLiveNodes() {
defer tab.mutex.Unlock()
now := time.Now()
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
for _, n := range b.entries {
if now.Sub(n.addedAt) >= seedMinTableTime {
tab.db.updateNode(n)
@ -524,7 +524,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
// obviously correct. I believe that tree-based buckets would make
// this easier to implement efficiently.
close := &nodesByDistance{target: target}
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
for _, n := range b.entries {
close.push(n, nresults)
}
@ -533,7 +533,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
}
func (tab *Table) len() (n int) {
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
n += len(b.entries)
}
return n

2
p2p/discv5/net_test.go
View file

@ -355,7 +355,7 @@ func (tn *preminedTestnet) mine(target NodeID) {
fmt.Printf(" target: %#v,\n", tn.target)
fmt.Printf(" targetSha: %#v,\n", tn.targetSha)
fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists))
for ld, ns := range tn.dists {
for ld, ns := range &tn.dists {
if len(ns) == 0 {
continue
}

8
p2p/discv5/table.go
View file

@ -81,7 +81,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
if printTable {
fmt.Println()
}
for i, b := range tab.buckets {
for i, b := range &tab.buckets {
entries += len(b.entries)
if printTable {
for _, e := range b.entries {
@ -93,7 +93,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash {
prefix := binary.BigEndian.Uint64(tab.self.sha[0:8])
dist := ^uint64(0)
entry := int(randUint(uint32(entries + 1)))
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
if entry < len(b.entries) {
n := b.entries[entry]
dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix
@ -121,7 +121,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) {
// TODO: tree-based buckets would help here
// Find all non-empty buckets and get a fresh slice of their entries.
var buckets [][]*Node
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
if len(b.entries) > 0 {
buckets = append(buckets, b.entries[:])
}
@ -175,7 +175,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance {
// obviously correct. I believe that tree-based buckets would make
// this easier to implement efficiently.
close := &nodesByDistance{target: target}
for _, b := range tab.buckets {
for _, b := range &tab.buckets {
for _, n := range b.entries {
close.push(n, nresults)
}

191
swarm/README.md
View file

@ -7,6 +7,21 @@ Swarm is a distributed storage platform and content distribution service, a nati
[![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum)
[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
## Table of Contents
* [Building the source](#building-the-source)
* [Running Swarm](#running-swarm)
* [Documentation](#documentation)
* [Developers Guide](#developers-guide)
* [Go Environment](#development-environment)
* [Vendored Dependencies](#vendored-dependencies)
* [Testing](#testing)
* [Profiling Swarm](#profiling-swarm)
* [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm)
* [Public Gateways](#public-gateways)
* [Swarm Dapps](#swarm-dapps)
* [Contributing](#contributing)
* [License](#license)
## Building the source
@ -16,13 +31,187 @@ Building Swarm requires Go (version 1.10 or later).
go install github.com/ethereum/go-ethereum/cmd/swarm
## Running Swarm
Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node.
To run Swarm you need an Ethereum account. You can create a new account by running the following command:
geth account new
You will be prompted for a password:
Your new account is locked with a password. Please give a password. Do not forget this password.
Passphrase:
Repeat passphrase:
Once you have specified the password, the output will be the Ethereum address representing that account. For example:
Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1}
Using this account, connect to Swarm with
swarm --bzzaccount <your-account-here>
# in our example
swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1
### Verifying that your local Swarm node is running
When running, Swarm is accessible through an HTTP API on port 8500.
Confirm that it is up and running by pointing your browser to http://localhost:8500
### Ethereum Name Service resolution
The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `geth` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag.
swarm --bzzaccount <your-account-here> \
--ens-api '$HOME/.ethereum/geth.ipc'
# in our example
swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \
--ens-api '$HOME/.ethereum/geth.ipc'
For more information on usage, features or command line flags, please consult the Documentation.
## Documentation
Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io).
## Contribution
## Developers Guide
### Go Environment
We assume that you have Go v1.10 installed, and `GOPATH` is set.
You must have your working copy under `$GOPATH/src/github.com/ethereum/go-ethereum`.
Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place:
```
git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ethereum/go-ethereum
```
### Vendored Dependencies
All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them.
If you want to add a new dependency, run `govendor fetch <import-path>`, then commit the result.
If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`.
### Testing
This section explains how to run unit, integration, and end-to-end tests in your development sandbox.
Testing one library:
```
go test -v -cpu 4 ./swarm/api
```
Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended.
Testing only some methods:
```
go test -v -cpu 4 ./eth -run TestMethod
```
Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both!
Running benchmarks:
```
go test -v -cpu 4 -bench . -run BenchmarkJoin
```
### Profiling Swarm
This section explains how to add Go `pprof` profiler to Swarm
If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060.
You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc.
By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging.
### Metrics and Instrumentation in Swarm
This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric.
Swarm metrics system is based on the `go-metrics` library.
The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types.
```
# incrementing a counter
metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1)
# measuring latency with a resetting timer
start := time.Now()
t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil)
...
t := UpdateSince(start)
```
#### Visualizing metrics
Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it:
```
swarm --help | grep metrics
```
We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own.
We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed.
Once you have `stateth` installed, and you have Docker running locally, you have to:
1. Run `stateth` and keep it running in the background
```
stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ethereum/go-ethereum/swarm/grafana_dashboards --influxdb-database metrics
```
2. Run `swarm` with at least the following params:
```
--metrics \
--metrics.influxdb.export \
--metrics.influxdb.endpoint "http://localhost:8086" \
--metrics.influxdb.username "admin" \
--metrics.influxdb.password "admin" \
--metrics.influxdb.database "metrics"
```
3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm.
## Public Gateways
Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node.
The Swarm public gateways are temporary and users should not rely on their existence for production services.
The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release.
## Swarm Dapps
You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth
Their source code can be found at: https://github.com/ethersphere/swarm-dapps
## Contributing
Thank you for considering to help out with the source code! We welcome contributions from
anyone on the internet, and are grateful for even the smallest of fixes!

3
swarm/api/api.go
View file

@ -339,8 +339,7 @@ func (a *API) Get(ctx context.Context, manifestAddr storage.Address, path string
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie error: %v", err))
return
return nil, "", http.StatusNotFound, nil, err
}
log.Debug("trie getting entry", "key", manifestAddr, "path", path)

1
swarm/api/config.go
View file

@ -63,6 +63,7 @@ type Config struct {
SwapEnabled bool
SyncEnabled bool
DeliverySkipCheck bool
LightNodeEnabled bool
SyncUpdateDelay time.Duration
SwapAPI string
Cors string

208
swarm/api/http/error.go
View file

@ -1,208 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
/*
Show nicely (but simple) formatted HTML error pages (or respond with JSON
if the appropriate `Accept` header is set)) for the http package.
*/
package http
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
"strings"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
l "github.com/ethereum/go-ethereum/swarm/log"
)
//templateMap holds a mapping of an HTTP error code to a template
var templateMap map[int]*template.Template
var caseErrors []CaseError
//metrics variables
var (
htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
)
//parameters needed for formatting the correct HTML page
type ResponseParams struct {
Msg string
Code int
Timestamp string
template *template.Template
Details template.HTML
}
//a custom error case struct that would be used to store validators and
//additional error info to display with client responses.
type CaseError struct {
Validator func(*Request) bool
Msg func(*Request) string
}
//we init the error handling right on boot time, so lookup and http response is fast
func init() {
initErrHandling()
}
func initErrHandling() {
//pages are saved as strings - get these strings
genErrPage := GetGenericErrorPage()
notFoundPage := GetNotFoundErrorPage()
multipleChoicesPage := GetMultipleChoicesErrorPage()
//map the codes to the available pages
tnames := map[int]string{
0: genErrPage, //default
http.StatusBadRequest: genErrPage,
http.StatusNotFound: notFoundPage,
http.StatusMultipleChoices: multipleChoicesPage,
http.StatusInternalServerError: genErrPage,
}
templateMap = make(map[int]*template.Template)
for code, tname := range tnames {
//assign formatted HTML to the code
templateMap[code] = template.Must(template.New(fmt.Sprintf("%d", code)).Parse(tname))
}
caseErrors = []CaseError{
{
Validator: func(r *Request) bool { return r.uri != nil && r.uri.Addr != "" && strings.HasPrefix(r.uri.Addr, "0x") },
Msg: func(r *Request) string {
uriCopy := r.uri
uriCopy.Addr = strings.TrimPrefix(uriCopy.Addr, "0x")
return fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
Please click <a href='%[1]s'>here</a> if your browser does not redirect you.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uriCopy.String())
},
}}
}
//ValidateCaseErrors is a method that process the request object through certain validators
//that assert if certain conditions are met for further information to log as an error
func ValidateCaseErrors(r *Request) string {
for _, err := range caseErrors {
if err.Validator(r) {
return err.Msg(r)
}
}
return ""
}
//ShowMultipeChoices is used when a user requests a resource in a manifest which results
//in ambiguous results. It returns a HTML page with clickable links of each of the entry
//in the manifest which fits the request URI ambiguity.
//For example, if the user requests bzz:/<hash>/read and that manifest contains entries
//"readme.md" and "readinglist.txt", a HTML page is returned with this two links.
//This only applies if the manifest has no default entry
func ShowMultipleChoices(w http.ResponseWriter, req *Request, list api.ManifestList) {
msg := ""
if list.Entries == nil {
Respond(w, req, "Could not resolve", http.StatusInternalServerError)
return
}
//make links relative
//requestURI comes with the prefix of the ambiguous path, e.g. "read" for "readme.md" and "readinglist.txt"
//to get clickable links, need to remove the ambiguous path, i.e. "read"
idx := strings.LastIndex(req.RequestURI, "/")
if idx == -1 {
Respond(w, req, "Internal Server Error", http.StatusInternalServerError)
return
}
//remove ambiguous part
base := req.RequestURI[:idx+1]
for _, e := range list.Entries {
//create clickable link for each entry
msg += "<a href='" + base + e.Path + "'>" + e.Path + "</a><br/>"
}
Respond(w, req, msg, http.StatusMultipleChoices)
}
//Respond is used to show an HTML page to a client.
//If there is an `Accept` header of `application/json`, JSON will be returned instead
//The function just takes a string message which will be displayed in the error page.
//The code is used to evaluate which template will be displayed
//(and return the correct HTTP status code)
func Respond(w http.ResponseWriter, req *Request, msg string, code int) {
additionalMessage := ValidateCaseErrors(req)
switch code {
case http.StatusInternalServerError:
log.Output(msg, log.LvlError, l.CallDepth, "ruid", req.ruid, "code", code)
case http.StatusMultipleChoices:
log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
listURI := api.URI{
Scheme: "bzz-list",
Addr: req.uri.Addr,
Path: req.uri.Path,
}
additionalMessage = fmt.Sprintf(`<a href="/%s">multiple choices</a>`, listURI.String())
default:
log.Output(msg, log.LvlDebug, l.CallDepth, "ruid", req.ruid, "code", code)
}
if code >= 400 {
w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
w.Header().Del("ETag")
}
respond(w, &req.Request, &ResponseParams{
Code: code,
Msg: msg,
Details: template.HTML(additionalMessage),
Timestamp: time.Now().Format(time.RFC1123),
template: getTemplate(code),
})
}
//evaluate if client accepts html or json response
func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
w.WriteHeader(params.Code)
if r.Header.Get("Accept") == "application/json" {
respondJSON(w, params)
} else {
respondHTML(w, params)
}
}
//return a HTML page
func respondHTML(w http.ResponseWriter, params *ResponseParams) {
htmlCounter.Inc(1)
err := params.template.Execute(w, params)
if err != nil {
log.Error(err.Error())
}
}
//return JSON
func respondJSON(w http.ResponseWriter, params *ResponseParams) {
jsonCounter.Inc(1)
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(params)
}
//get the HTML template for a given code
func getTemplate(code int) *template.Template {
if val, tmpl := templateMap[code]; tmpl {
return val
}
return templateMap[0]
}

559
swarm/api/http/error_templates.go
View file

File diff suppressed because one or more lines are too long

95
swarm/api/http/middleware.go Normal file
View file

@ -0,0 +1,95 @@
package http
import (
"fmt"
"net/http"
"runtime/debug"
"strings"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/pborman/uuid"
)
// Adapt chains h (main request handler) main handler to adapters (middleware handlers)
// Please note that the order of execution for `adapters` is FIFO (adapters[0] will be executed first)
func Adapt(h http.Handler, adapters ...Adapter) http.Handler {
for i := range adapters {
adapter := adapters[len(adapters)-1-i]
h = adapter(h)
}
return h
}
type Adapter func(http.Handler) http.Handler
func SetRequestID(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r = r.WithContext(SetRUID(r.Context(), uuid.New()[:8]))
metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
log.Info("created ruid for request", "ruid", GetRUID(r.Context()), "method", r.Method, "url", r.RequestURI)
h.ServeHTTP(w, r)
})
}
func ParseURI(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
if err != nil {
w.WriteHeader(http.StatusBadRequest)
RespondError(w, r, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
return
}
if uri.Addr != "" && strings.HasPrefix(uri.Addr, "0x") {
uri.Addr = strings.TrimPrefix(uri.Addr, "0x")
msg := fmt.Sprintf(`The requested hash seems to be prefixed with '0x'. You will be redirected to the correct URL within 5 seconds.<br/>
Please click <a href='%[1]s'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout("location.href='%[1]s';",5000);</script>`, "/"+uri.String())
w.WriteHeader(http.StatusNotFound)
w.Write([]byte(msg))
return
}
ctx := r.Context()
r = r.WithContext(SetURI(ctx, uri))
log.Debug("parsed request path", "ruid", GetRUID(r.Context()), "method", r.Method, "uri.Addr", uri.Addr, "uri.Path", uri.Path, "uri.Scheme", uri.Scheme)
h.ServeHTTP(w, r)
})
}
func InitLoggingResponseWriter(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
writer := newLoggingResponseWriter(w)
h.ServeHTTP(writer, r)
})
}
func InstrumentOpenTracing(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
uri := GetURI(r.Context())
if uri == nil || r.Method == "" || (uri != nil && uri.Scheme == "") {
h.ServeHTTP(w, r) // soft fail
return
}
spanName := fmt.Sprintf("http.%s.%s", r.Method, uri.Scheme)
ctx, sp := spancontext.StartSpan(r.Context(), spanName)
defer sp.Finish()
h.ServeHTTP(w, r.WithContext(ctx))
})
}
func RecoverPanic(h http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if err := recover(); err != nil {
log.Error("panic recovery!", "stack trace", debug.Stack(), "url", r.URL.String(), "headers", r.Header)
}
}()
h.ServeHTTP(w, r)
})
}

132
swarm/api/http/response.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package http
import (
"encoding/json"
"fmt"
"html/template"
"net/http"
"strings"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
)
var (
htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil)
jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil)
plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil)
)
type ResponseParams struct {
Msg template.HTML
Code int
Timestamp string
template *template.Template
Details template.HTML
}
// ShowMultipleChoices is used when a user requests a resource in a manifest which results
// in ambiguous results. It returns a HTML page with clickable links of each of the entry
// in the manifest which fits the request URI ambiguity.
// For example, if the user requests bzz:/<hash>/read and that manifest contains entries
// "readme.md" and "readinglist.txt", a HTML page is returned with this two links.
// This only applies if the manifest has no default entry
func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) {
log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
msg := ""
if list.Entries == nil {
RespondError(w, r, "Could not resolve", http.StatusInternalServerError)
return
}
requestUri := strings.TrimPrefix(r.RequestURI, "/")
uri, err := api.Parse(requestUri)
if err != nil {
RespondError(w, r, "Bad Request", http.StatusBadRequest)
}
uri.Scheme = "bzz-list"
msg += fmt.Sprintf("Disambiguation:<br/>Your request may refer to multiple choices.<br/>Click <a class=\"orange\" href='"+"/"+uri.String()+"'>here</a> if your browser does not redirect you within 5 seconds.<script>setTimeout(\"location.href='%s';\",5000);</script><br/>", "/"+uri.String())
RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices)
}
func RespondTemplate(w http.ResponseWriter, r *http.Request, templateName, msg string, code int) {
log.Debug("RespondTemplate", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
respond(w, r, &ResponseParams{
Code: code,
Msg: template.HTML(msg),
Timestamp: time.Now().Format(time.RFC1123),
template: TemplatesMap[templateName],
})
}
func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) {
log.Debug("RespondError", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context()))
RespondTemplate(w, r, "error", msg, code)
}
func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
w.WriteHeader(params.Code)
if params.Code >= 400 {
w.Header().Del("Cache-Control") //avoid sending cache headers for errors!
w.Header().Del("ETag")
}
acceptHeader := r.Header.Get("Accept")
// this cannot be in a switch form since an Accept header can be in the form of "Accept: */*, text/html, application/xhtml+xml, application/xml;q=0.9, */*;q=0.8"
if strings.Contains(acceptHeader, "application/json") {
if err := respondJSON(w, r, params); err != nil {
RespondError(w, r, "Internal server error", http.StatusInternalServerError)
}
} else if strings.Contains(acceptHeader, "text/html") {
respondHTML(w, r, params)
} else {
respondPlaintext(w, r, params) //returns nice errors for curl
}
}
func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) {
htmlCounter.Inc(1)
log.Debug("respondHTML", "ruid", GetRUID(r.Context()))
err := params.template.Execute(w, params)
if err != nil {
log.Error(err.Error())
}
}
func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
jsonCounter.Inc(1)
log.Debug("respondJSON", "ruid", GetRUID(r.Context()))
w.Header().Set("Content-Type", "application/json")
return json.NewEncoder(w).Encode(params)
}
func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error {
plaintextCounter.Inc(1)
log.Debug("respondPlaintext", "ruid", GetRUID(r.Context()))
w.Header().Set("Content-Type", "text/plain")
strToWrite := "Code: " + fmt.Sprintf("%d", params.Code) + "\n"
strToWrite += "Message: " + string(params.Msg) + "\n"
strToWrite += "Timestamp: " + params.Timestamp + "\n"
_, err := w.Write([]byte(strToWrite))
return err
}

2
swarm/api/http/error_test.go → swarm/api/http/response_test.go
View file

@ -44,7 +44,7 @@ func TestError(t *testing.T) {
defer resp.Body.Close()
respbody, err = ioutil.ReadAll(resp.Body)
if resp.StatusCode != 400 && !strings.Contains(string(respbody), "Invalid URI &#34;/this_should_fail_as_no_bzz_protocol_present&#34;: unknown scheme") {
if resp.StatusCode != 404 && !strings.Contains(string(respbody), "Invalid URI &#34;/this_should_fail_as_no_bzz_protocol_present&#34;: unknown scheme") {
t.Fatalf("Response body does not match, expected: %v, to contain: %v; received code %d, expected code: %d", string(respbody), "Invalid bzz URI: unknown scheme", 400, resp.StatusCode)
}

38
swarm/api/http/sctx.go Normal file
View file

@ -0,0 +1,38 @@
package http
import (
"context"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/sctx"
)
type contextKey int
const (
uriKey contextKey = iota
)
func GetRUID(ctx context.Context) string {
v, ok := ctx.Value(sctx.HTTPRequestIDKey).(string)
if ok {
return v
}
return "xxxxxxxx"
}
func SetRUID(ctx context.Context, ruid string) context.Context {
return context.WithValue(ctx, sctx.HTTPRequestIDKey, ruid)
}
func GetURI(ctx context.Context) *api.URI {
v, ok := ctx.Value(uriKey).(*api.URI)
if ok {
return v
}
return nil
}
func SetURI(ctx context.Context, uri *api.URI) context.Context {
return context.WithValue(ctx, uriKey, uri)
}

563
swarm/api/http/server.go
View file

@ -41,12 +41,9 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/api"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/mru"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pborman/uuid"
"github.com/rs/cors"
)
@ -72,6 +69,17 @@ var (
getListFail = metrics.NewRegisteredCounter("api.http.get.list.fail", nil)
)
type methodHandler map[string]http.Handler
func (m methodHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
v, ok := m[r.Method]
if ok {
v.ServeHTTP(rw, r)
return
}
rw.WriteHeader(http.StatusMethodNotAllowed)
}
func NewServer(api *api.API, corsString string) *Server {
var allowedOrigins []string
for _, domain := range strings.Split(corsString, ",") {
@ -84,20 +92,79 @@ func NewServer(api *api.API, corsString string) *Server {
AllowedHeaders: []string{"*"},
})
mux := http.NewServeMux()
server := &Server{api: api}
mux.HandleFunc("/bzz:/", server.WrapHandler(true, server.HandleBzz))
mux.HandleFunc("/bzz-raw:/", server.WrapHandler(true, server.HandleBzzRaw))
mux.HandleFunc("/bzz-immutable:/", server.WrapHandler(true, server.HandleBzzImmutable))
mux.HandleFunc("/bzz-hash:/", server.WrapHandler(true, server.HandleBzzHash))
mux.HandleFunc("/bzz-list:/", server.WrapHandler(true, server.HandleBzzList))
mux.HandleFunc("/bzz-resource:/", server.WrapHandler(true, server.HandleBzzResource))
mux.HandleFunc("/", server.WrapHandler(false, server.HandleRootPaths))
mux.HandleFunc("/robots.txt", server.WrapHandler(false, server.HandleRootPaths))
mux.HandleFunc("/favicon.ico", server.WrapHandler(false, server.HandleRootPaths))
defaultMiddlewares := []Adapter{
RecoverPanic,
SetRequestID,
InitLoggingResponseWriter,
ParseURI,
InstrumentOpenTracing,
}
mux := http.NewServeMux()
mux.Handle("/bzz:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleBzzGet),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostFiles),
defaultMiddlewares...,
),
"DELETE": Adapt(
http.HandlerFunc(server.HandleDelete),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-raw:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGet),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostRaw),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-immutable:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGet),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-hash:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGet),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-list:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGetList),
defaultMiddlewares...,
),
})
mux.Handle("/bzz-resource:/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleGetResource),
defaultMiddlewares...,
),
"POST": Adapt(
http.HandlerFunc(server.HandlePostResource),
defaultMiddlewares...,
),
})
mux.Handle("/", methodHandler{
"GET": Adapt(
http.HandlerFunc(server.HandleRootPaths),
SetRequestID,
InitLoggingResponseWriter,
),
})
server.Handler = c.Handler(mux)
return server
}
@ -105,139 +172,6 @@ func (s *Server) ListenAndServe(addr string) error {
return http.ListenAndServe(addr, s)
}
func (s *Server) HandleRootPaths(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
if r.RequestURI == "/" {
if strings.Contains(r.Header.Get("Accept"), "text/html") {
err := landingPageTemplate.Execute(w, nil)
if err != nil {
log.Error(fmt.Sprintf("error rendering landing page: %s", err))
}
return
}
if strings.Contains(r.Header.Get("Accept"), "application/json") {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
json.NewEncoder(w).Encode("Welcome to Swarm!")
return
}
}
if r.URL.Path == "/robots.txt" {
w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
fmt.Fprintf(w, "User-agent: *\nDisallow: /")
return
}
Respond(w, r, "Bad Request", http.StatusBadRequest)
default:
Respond(w, r, "Not Found", http.StatusNotFound)
}
}
func (s *Server) HandleBzz(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetBzz")
if r.Header.Get("Accept") == "application/x-tar" {
reader, err := s.api.GetDirectoryTar(r.Context(), r.uri)
if err != nil {
Respond(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
}
defer reader.Close()
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
io.Copy(w, reader)
return
}
s.HandleGetFile(w, r)
case http.MethodPost:
log.Debug("handlePostFiles")
s.HandlePostFiles(w, r)
case http.MethodDelete:
log.Debug("handleBzzDelete")
s.HandleDelete(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) HandleBzzRaw(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetRaw")
s.HandleGet(w, r)
case http.MethodPost:
log.Debug("handlePostRaw")
s.HandlePostRaw(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) HandleBzzImmutable(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetHash")
s.HandleGetList(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) HandleBzzHash(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetHash")
s.HandleGet(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) HandleBzzList(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetHash")
s.HandleGetList(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) HandleBzzResource(w http.ResponseWriter, r *Request) {
switch r.Method {
case http.MethodGet:
log.Debug("handleGetResource")
s.HandleGetResource(w, r)
case http.MethodPost:
log.Debug("handlePostResource")
s.HandlePostResource(w, r)
default:
Respond(w, r, "Method not allowed", http.StatusMethodNotAllowed)
}
}
func (s *Server) WrapHandler(parseBzzUri bool, h func(http.ResponseWriter, *Request)) http.HandlerFunc {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
defer metrics.GetOrRegisterResettingTimer(fmt.Sprintf("http.request.%s.time", r.Method), nil).UpdateSince(time.Now())
req := &Request{Request: *r, ruid: uuid.New()[:8]}
metrics.GetOrRegisterCounter(fmt.Sprintf("http.request.%s", r.Method), nil).Inc(1)
log.Info("serving request", "ruid", req.ruid, "method", r.Method, "url", r.RequestURI)
// wrapping the ResponseWriter, so that we get the response code set by http.ServeContent
w := newLoggingResponseWriter(rw)
if parseBzzUri {
uri, err := api.Parse(strings.TrimLeft(r.URL.Path, "/"))
if err != nil {
Respond(w, req, fmt.Sprintf("invalid URI %q", r.URL.Path), http.StatusBadRequest)
return
}
req.uri = uri
log.Debug("parsed request path", "ruid", req.ruid, "method", req.Method, "uri.Addr", req.uri.Addr, "uri.Path", req.uri.Path, "uri.Scheme", req.uri.Scheme)
}
h(w, req) // call original
log.Info("served response", "ruid", req.ruid, "code", w.statusCode)
})
}
// browser API for registering bzz url scheme handlers:
// https://developer.mozilla.org/en/docs/Web-based_protocol_handlers
// electron (chromium) api for registering bzz url scheme handlers:
@ -247,59 +181,81 @@ type Server struct {
api *api.API
}
// Request wraps http.Request and also includes the parsed bzz URI
type Request struct {
http.Request
func (s *Server) HandleBzzGet(w http.ResponseWriter, r *http.Request) {
log.Debug("handleBzzGet", "ruid", GetRUID(r.Context()))
if r.Header.Get("Accept") == "application/x-tar" {
uri := GetURI(r.Context())
reader, err := s.api.GetDirectoryTar(r.Context(), uri)
if err != nil {
RespondError(w, r, fmt.Sprintf("Had an error building the tarball: %v", err), http.StatusInternalServerError)
}
defer reader.Close()
uri *api.URI
ruid string // request unique id
w.Header().Set("Content-Type", "application/x-tar")
w.WriteHeader(http.StatusOK)
io.Copy(w, reader)
return
}
s.HandleGetFile(w, r)
}
func (s *Server) HandleRootPaths(w http.ResponseWriter, r *http.Request) {
switch r.RequestURI {
case "/":
RespondTemplate(w, r, "landing-page", "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", 200)
return
case "/robots.txt":
w.Header().Set("Last-Modified", time.Now().Format(http.TimeFormat))
fmt.Fprintf(w, "User-agent: *\nDisallow: /")
case "/favicon.ico":
w.WriteHeader(http.StatusOK)
w.Write(faviconBytes)
default:
RespondError(w, r, "Not Found", http.StatusNotFound)
}
}
// HandlePostRaw handles a POST request to a raw bzz-raw:/ URI, stores the request
// body in swarm and returns the resulting storage address as a text/plain response
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
log.Debug("handle.post.raw", "ruid", r.ruid)
func (s *Server) HandlePostRaw(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
log.Debug("handle.post.raw", "ruid", ruid)
postRawCount.Inc(1)
ctx := r.Context()
var sp opentracing.Span
ctx, sp = spancontext.StartSpan(
ctx,
"http.post.raw")
defer sp.Finish()
toEncrypt := false
if r.uri.Addr == "encrypt" {
uri := GetURI(r.Context())
if uri.Addr == "encrypt" {
toEncrypt = true
}
if r.uri.Path != "" {
if uri.Path != "" {
postRawFail.Inc(1)
Respond(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
RespondError(w, r, "raw POST request cannot contain a path", http.StatusBadRequest)
return
}
if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
if uri.Addr != "" && uri.Addr != "encrypt" {
postRawFail.Inc(1)
Respond(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
RespondError(w, r, "raw POST request addr can only be empty or \"encrypt\"", http.StatusBadRequest)
return
}
if r.Header.Get("Content-Length") == "" {
postRawFail.Inc(1)
Respond(w, r, "missing Content-Length header in request", http.StatusBadRequest)
RespondError(w, r, "missing Content-Length header in request", http.StatusBadRequest)
return
}
addr, _, err := s.api.Store(ctx, r.Body, r.ContentLength, toEncrypt)
addr, _, err := s.api.Store(r.Context(), r.Body, r.ContentLength, toEncrypt)
if err != nil {
postRawFail.Inc(1)
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug("stored content", "ruid", r.ruid, "key", addr)
log.Debug("stored content", "ruid", ruid, "key", addr)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
@ -311,55 +267,49 @@ func (s *Server) HandlePostRaw(w http.ResponseWriter, r *Request) {
// (either a tar archive or multipart form), adds those files either to an
// existing manifest or to a new manifest under <path> and returns the
// resulting manifest hash as a text/plain response
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
log.Debug("handle.post.files", "ruid", r.ruid)
func (s *Server) HandlePostFiles(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
log.Debug("handle.post.files", "ruid", ruid)
postFilesCount.Inc(1)
var sp opentracing.Span
ctx := r.Context()
ctx, sp = spancontext.StartSpan(
ctx,
"http.post.files")
defer sp.Finish()
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
if err != nil {
postFilesFail.Inc(1)
Respond(w, r, err.Error(), http.StatusBadRequest)
RespondError(w, r, err.Error(), http.StatusBadRequest)
return
}
toEncrypt := false
if r.uri.Addr == "encrypt" {
uri := GetURI(r.Context())
if uri.Addr == "encrypt" {
toEncrypt = true
}
var addr storage.Address
if r.uri.Addr != "" && r.uri.Addr != "encrypt" {
addr, err = s.api.Resolve(r.Context(), r.uri)
if uri.Addr != "" && uri.Addr != "encrypt" {
addr, err = s.api.Resolve(r.Context(), uri)
if err != nil {
postFilesFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusInternalServerError)
return
}
log.Debug("resolved key", "ruid", r.ruid, "key", addr)
log.Debug("resolved key", "ruid", ruid, "key", addr)
} else {
addr, err = s.api.NewManifest(r.Context(), toEncrypt)
if err != nil {
postFilesFail.Inc(1)
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug("new manifest", "ruid", r.ruid, "key", addr)
log.Debug("new manifest", "ruid", ruid, "key", addr)
}
newAddr, err := s.api.UpdateManifest(ctx, addr, func(mw *api.ManifestWriter) error {
newAddr, err := s.api.UpdateManifest(r.Context(), addr, func(mw *api.ManifestWriter) error {
switch contentType {
case "application/x-tar":
_, err := s.handleTarUpload(r, mw)
if err != nil {
Respond(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("error uploading tarball: %v", err), http.StatusInternalServerError)
return err
}
return nil
@ -372,30 +322,31 @@ func (s *Server) HandlePostFiles(w http.ResponseWriter, r *Request) {
})
if err != nil {
postFilesFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("cannot create manifest: %s", err), http.StatusInternalServerError)
return
}
log.Debug("stored content", "ruid", r.ruid, "key", newAddr)
log.Debug("stored content", "ruid", ruid, "key", newAddr)
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, newAddr)
}
func (s *Server) handleTarUpload(r *Request, mw *api.ManifestWriter) (storage.Address, error) {
log.Debug("handle.tar.upload", "ruid", r.ruid)
func (s *Server) handleTarUpload(r *http.Request, mw *api.ManifestWriter) (storage.Address, error) {
log.Debug("handle.tar.upload", "ruid", GetRUID(r.Context()))
key, err := s.api.UploadTar(r.Context(), r.Body, r.uri.Path, mw)
key, err := s.api.UploadTar(r.Context(), r.Body, GetURI(r.Context()).Path, mw)
if err != nil {
return nil, err
}
return key, nil
}
func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.ManifestWriter) error {
log.Debug("handle.multipart.upload", "ruid", req.ruid)
mr := multipart.NewReader(req.Body, boundary)
func (s *Server) handleMultipartUpload(r *http.Request, boundary string, mw *api.ManifestWriter) error {
ruid := GetRUID(r.Context())
log.Debug("handle.multipart.upload", "ruid", ruid)
mr := multipart.NewReader(r.Body, boundary)
for {
part, err := mr.NextPart()
if err == io.EOF {
@ -435,48 +386,52 @@ func (s *Server) handleMultipartUpload(req *Request, boundary string, mw *api.Ma
if name == "" {
name = part.FormName()
}
path := path.Join(req.uri.Path, name)
uri := GetURI(r.Context())
path := path.Join(uri.Path, name)
entry := &api.ManifestEntry{
Path: path,
ContentType: part.Header.Get("Content-Type"),
Size: size,
ModTime: time.Now(),
}
log.Debug("adding path to new manifest", "ruid", req.ruid, "bytes", entry.Size, "path", entry.Path)
contentKey, err := mw.AddEntry(req.Context(), reader, entry)
log.Debug("adding path to new manifest", "ruid", ruid, "bytes", entry.Size, "path", entry.Path)
contentKey, err := mw.AddEntry(r.Context(), reader, entry)
if err != nil {
return fmt.Errorf("error adding manifest entry from multipart form: %s", err)
}
log.Debug("stored content", "ruid", req.ruid, "key", contentKey)
log.Debug("stored content", "ruid", ruid, "key", contentKey)
}
}
func (s *Server) handleDirectUpload(req *Request, mw *api.ManifestWriter) error {
log.Debug("handle.direct.upload", "ruid", req.ruid)
key, err := mw.AddEntry(req.Context(), req.Body, &api.ManifestEntry{
Path: req.uri.Path,
ContentType: req.Header.Get("Content-Type"),
func (s *Server) handleDirectUpload(r *http.Request, mw *api.ManifestWriter) error {
ruid := GetRUID(r.Context())
log.Debug("handle.direct.upload", "ruid", ruid)
key, err := mw.AddEntry(r.Context(), r.Body, &api.ManifestEntry{
Path: GetURI(r.Context()).Path,
ContentType: r.Header.Get("Content-Type"),
Mode: 0644,
Size: req.ContentLength,
Size: r.ContentLength,
ModTime: time.Now(),
})
if err != nil {
return err
}
log.Debug("stored content", "ruid", req.ruid, "key", key)
log.Debug("stored content", "ruid", ruid, "key", key)
return nil
}
// HandleDelete handles a DELETE request to bzz:/<manifest>/<path>, removes
// <path> from <manifest> and returns the resulting manifest hash as a
// text/plain response
func (s *Server) HandleDelete(w http.ResponseWriter, r *Request) {
log.Debug("handle.delete", "ruid", r.ruid)
func (s *Server) HandleDelete(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.delete", "ruid", ruid)
deleteCount.Inc(1)
newKey, err := s.api.Delete(r.Context(), r.uri.Addr, r.uri.Path)
newKey, err := s.api.Delete(r.Context(), uri.Addr, uri.Path)
if err != nil {
deleteFail.Inc(1)
Respond(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("could not delete from manifest: %v", err), http.StatusInternalServerError)
return
}
@ -519,27 +474,20 @@ func resourcePostMode(path string) (isRaw bool, frequency uint64, err error) {
//
// The POST request admits a JSON structure as defined in the mru package: `mru.updateRequestJSON`
// The requests can be to a) create a resource, b) update a resource or c) both a+b: create a resource and set the initial content
func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
log.Debug("handle.post.resource", "ruid", r.ruid)
var sp opentracing.Span
ctx := r.Context()
ctx, sp = spancontext.StartSpan(
ctx,
"http.post.resource")
defer sp.Finish()
func (s *Server) HandlePostResource(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
log.Debug("handle.post.resource", "ruid", ruid)
var err error
// Creation and update must send mru.updateRequestJSON JSON structure
body, err := ioutil.ReadAll(r.Body)
if err != nil {
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
var updateRequest mru.Request
if err := updateRequest.UnmarshalJSON(body); err != nil { // decodes request JSON
Respond(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
RespondError(w, r, err.Error(), http.StatusBadRequest) //TODO: send different status response depending on error
return
}
@ -548,7 +496,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
// to update this resource
// Check this early, to avoid creating a resource and then not being able to set its first update.
if err = updateRequest.Verify(); err != nil {
Respond(w, r, err.Error(), http.StatusForbidden)
RespondError(w, r, err.Error(), http.StatusForbidden)
return
}
}
@ -557,7 +505,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
err = s.api.ResourceCreate(r.Context(), &updateRequest)
if err != nil {
code, err2 := s.translateResourceError(w, r, "resource creation fail", err)
Respond(w, r, err2.Error(), code)
RespondError(w, r, err2.Error(), code)
return
}
}
@ -565,7 +513,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
if updateRequest.IsUpdate() {
_, err = s.api.ResourceUpdate(r.Context(), &updateRequest.SignedResourceUpdate)
if err != nil {
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
}
@ -579,7 +527,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
// metadata chunk (rootAddr)
m, err := s.api.NewResourceManifest(r.Context(), updateRequest.RootAddr().Hex())
if err != nil {
Respond(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("failed to create resource manifest: %v", err), http.StatusInternalServerError)
return
}
@ -589,7 +537,7 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
// \TODO update manifest key automatically in ENS
outdata, err := json.Marshal(m)
if err != nil {
Respond(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("failed to create json response: %s", err), http.StatusInternalServerError)
return
}
fmt.Fprint(w, string(outdata))
@ -604,17 +552,19 @@ func (s *Server) HandlePostResource(w http.ResponseWriter, r *Request) {
// bzz-resource://<id>/meta - get metadata and next version information
// <id> = ens name or hash
// TODO: Enable pass maxPeriod parameter
func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
log.Debug("handle.get.resource", "ruid", r.ruid)
func (s *Server) HandleGetResource(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get.resource", "ruid", ruid)
var err error
// resolve the content key.
manifestAddr := r.uri.Address()
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
manifestAddr, err = s.api.Resolve(r.Context(), uri)
if err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
@ -625,25 +575,25 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
rootAddr, err := s.api.ResolveResourceManifest(r.Context(), manifestAddr)
if err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", r.uri.Addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("error resolving resource root chunk for %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
log.Debug("handle.get.resource: resolved", "ruid", r.ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
log.Debug("handle.get.resource: resolved", "ruid", ruid, "manifestkey", manifestAddr, "rootchunk addr", rootAddr)
// determine if the query specifies period and version or it is a metadata query
var params []string
if len(r.uri.Path) > 0 {
if r.uri.Path == "meta" {
if len(uri.Path) > 0 {
if uri.Path == "meta" {
unsignedUpdateRequest, err := s.api.ResourceNewRequest(r.Context(), rootAddr)
if err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("cannot retrieve resource metadata for rootAddr=%s: %s", rootAddr.Hex(), err), http.StatusNotFound)
return
}
rawResponse, err := unsignedUpdateRequest.MarshalJSON()
if err != nil {
Respond(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
RespondError(w, r, fmt.Sprintf("cannot encode unsigned UpdateRequest: %v", err), http.StatusInternalServerError)
return
}
w.Header().Add("Content-type", "application/json")
@ -653,7 +603,7 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
}
params = strings.Split(r.uri.Path, "/")
params = strings.Split(uri.Path, "/")
}
var name string
@ -689,17 +639,17 @@ func (s *Server) HandleGetResource(w http.ResponseWriter, r *Request) {
// any error from the switch statement will end up here
if err != nil {
code, err2 := s.translateResourceError(w, r, "mutable resource lookup fail", err)
Respond(w, r, err2.Error(), code)
RespondError(w, r, err2.Error(), code)
return
}
// All ok, serve the retrieved update
log.Debug("Found update", "name", name, "ruid", r.ruid)
log.Debug("Found update", "name", name, "ruid", ruid)
w.Header().Set("Content-Type", "application/octet-stream")
http.ServeContent(w, &r.Request, "", now, bytes.NewReader(data))
http.ServeContent(w, r, "", now, bytes.NewReader(data))
}
func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supErr string, err error) (int, error) {
func (s *Server) translateResourceError(w http.ResponseWriter, r *http.Request, supErr string, err error) (int, error) {
code := 0
defaultErr := fmt.Errorf("%s: %v", supErr, err)
rsrcErr, ok := err.(*mru.Error)
@ -725,46 +675,41 @@ func (s *Server) translateResourceError(w http.ResponseWriter, r *Request, supEr
// given storage key
// - bzz-hash://<key> and responds with the hash of the content stored
// at the given storage key as a text/plain response
func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
log.Debug("handle.get", "ruid", r.ruid, "uri", r.uri)
func (s *Server) HandleGet(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get", "ruid", ruid, "uri", uri)
getCount.Inc(1)
var sp opentracing.Span
ctx := r.Context()
ctx, sp = spancontext.StartSpan(
ctx,
"http.get")
defer sp.Finish()
var err error
addr := r.uri.Address()
addr := uri.Address()
if addr == nil {
addr, err = s.api.Resolve(r.Context(), r.uri)
addr, err = s.api.Resolve(r.Context(), uri)
if err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
}
log.Debug("handle.get: resolved", "ruid", r.ruid, "key", addr)
log.Debug("handle.get: resolved", "ruid", ruid, "key", addr)
// if path is set, interpret <key> as a manifest and return the
// raw entry at the given path
if r.uri.Path != "" {
if uri.Path != "" {
walker, err := s.api.NewManifestWalker(r.Context(), addr, nil)
if err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
RespondError(w, r, fmt.Sprintf("%s is not a manifest", addr), http.StatusBadRequest)
return
}
var entry *api.ManifestEntry
walker.Walk(func(e *api.ManifestEntry) error {
// if the entry matches the path, set entry and stop
// the walk
if e.Path == r.uri.Path {
if e.Path == uri.Path {
entry = e
// return an error to cancel the walk
return errors.New("found")
@ -778,7 +723,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
// if the manifest's path is a prefix of the
// requested path, recurse into it by returning
// nil and continuing the walk
if strings.HasPrefix(r.uri.Path, e.Path) {
if strings.HasPrefix(uri.Path, e.Path) {
return nil
}
@ -786,7 +731,7 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
})
if entry == nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("manifest entry could not be loaded"), http.StatusNotFound)
return
}
addr = storage.Address(common.Hex2Bytes(entry.Hash))
@ -796,23 +741,23 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to manifest key or raw entry key.
if noneMatchEtag != "" {
if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), addr) {
Respond(w, r, "Not Modified", http.StatusNotModified)
w.WriteHeader(http.StatusNotModified)
return
}
}
// check the root chunk exists by retrieving the file's size
reader, isEncrypted := s.api.Retrieve(ctx, addr)
if _, err := reader.Size(ctx, nil); err != nil {
reader, isEncrypted := s.api.Retrieve(r.Context(), addr)
if _, err := reader.Size(r.Context(), nil); err != nil {
getFail.Inc(1)
Respond(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("root chunk not found %s: %s", addr, err), http.StatusNotFound)
return
}
w.Header().Set("X-Decrypted", fmt.Sprintf("%v", isEncrypted))
switch {
case r.uri.Raw():
case uri.Raw():
// allow the request to overwrite the content type using a query
// parameter
contentType := "application/octet-stream"
@ -820,8 +765,8 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
contentType = typ
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), reader)
case r.uri.Hash():
http.ServeContent(w, r, "", time.Now(), reader)
case uri.Hash():
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, addr)
@ -831,35 +776,30 @@ func (s *Server) HandleGet(w http.ResponseWriter, r *Request) {
// HandleGetList handles a GET request to bzz-list:/<manifest>/<path> and returns
// a list of all files contained in <manifest> under <path> grouped into
// common prefixes using "/" as a delimiter
func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
log.Debug("handle.get.list", "ruid", r.ruid, "uri", r.uri)
func (s *Server) HandleGetList(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get.list", "ruid", ruid, "uri", uri)
getListCount.Inc(1)
var sp opentracing.Span
ctx := r.Context()
ctx, sp = spancontext.StartSpan(
ctx,
"http.get.list")
defer sp.Finish()
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
addr, err := s.api.Resolve(r.Context(), r.uri)
addr, err := s.api.Resolve(r.Context(), uri)
if err != nil {
getListFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
log.Debug("handle.get.list: resolved", "ruid", r.ruid, "key", addr)
log.Debug("handle.get.list: resolved", "ruid", ruid, "key", addr)
list, err := s.api.GetManifestList(ctx, addr, r.uri.Path)
list, err := s.api.GetManifestList(r.Context(), addr, uri.Path)
if err != nil {
getListFail.Inc(1)
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
@ -867,11 +807,11 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
// HTML index with relative URLs
if strings.Contains(r.Header.Get("Accept"), "text/html") {
w.Header().Set("Content-Type", "text/html")
err := htmlListTemplate.Execute(w, &htmlListData{
err := TemplatesMap["bzz-list"].Execute(w, &htmlListData{
URI: &api.URI{
Scheme: "bzz",
Addr: r.uri.Addr,
Path: r.uri.Path,
Addr: uri.Addr,
Path: uri.Path,
},
List: &list,
})
@ -888,45 +828,40 @@ func (s *Server) HandleGetList(w http.ResponseWriter, r *Request) {
// HandleGetFile handles a GET request to bzz://<manifest>/<path> and responds
// with the content of the file at <path> from the given <manifest>
func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
log.Debug("handle.get.file", "ruid", r.ruid)
func (s *Server) HandleGetFile(w http.ResponseWriter, r *http.Request) {
ruid := GetRUID(r.Context())
uri := GetURI(r.Context())
log.Debug("handle.get.file", "ruid", ruid)
getFileCount.Inc(1)
var sp opentracing.Span
ctx := r.Context()
ctx, sp = spancontext.StartSpan(
ctx,
"http.get.file")
defer sp.Finish()
// ensure the root path has a trailing slash so that relative URLs work
if r.uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, &r.Request, r.URL.Path+"/", http.StatusMovedPermanently)
if uri.Path == "" && !strings.HasSuffix(r.URL.Path, "/") {
http.Redirect(w, r, r.URL.Path+"/", http.StatusMovedPermanently)
return
}
var err error
manifestAddr := r.uri.Address()
manifestAddr := uri.Address()
if manifestAddr == nil {
manifestAddr, err = s.api.Resolve(r.Context(), r.uri)
manifestAddr, err = s.api.Resolve(r.Context(), uri)
if err != nil {
getFileFail.Inc(1)
Respond(w, r, fmt.Sprintf("cannot resolve %s: %s", r.uri.Addr, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("cannot resolve %s: %s", uri.Addr, err), http.StatusNotFound)
return
}
} else {
w.Header().Set("Cache-Control", "max-age=2147483648, immutable") // url was of type bzz://<hex key>/path, so we are sure it is immutable.
}
log.Debug("handle.get.file: resolved", "ruid", r.ruid, "key", manifestAddr)
reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, r.uri.Path)
log.Debug("handle.get.file: resolved", "ruid", ruid, "key", manifestAddr)
reader, contentType, status, contentKey, err := s.api.Get(r.Context(), manifestAddr, uri.Path)
etag := common.Bytes2Hex(contentKey)
noneMatchEtag := r.Header.Get("If-None-Match")
w.Header().Set("ETag", fmt.Sprintf("%q", etag)) // set etag to actual content key.
if noneMatchEtag != "" {
if bytes.Equal(storage.Address(common.Hex2Bytes(noneMatchEtag)), contentKey) {
Respond(w, r, "Not Modified", http.StatusNotModified)
w.WriteHeader(http.StatusNotModified)
return
}
}
@ -935,10 +870,10 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
switch status {
case http.StatusNotFound:
getFileNotFound.Inc(1)
Respond(w, r, err.Error(), http.StatusNotFound)
RespondError(w, r, err.Error(), http.StatusNotFound)
default:
getFileFail.Inc(1)
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
}
return
}
@ -946,28 +881,28 @@ func (s *Server) HandleGetFile(w http.ResponseWriter, r *Request) {
//the request results in ambiguous files
//e.g. /read with readme.md and readinglist.txt available in manifest
if status == http.StatusMultipleChoices {
list, err := s.api.GetManifestList(ctx, manifestAddr, r.uri.Path)
list, err := s.api.GetManifestList(r.Context(), manifestAddr, uri.Path)
if err != nil {
getFileFail.Inc(1)
Respond(w, r, err.Error(), http.StatusInternalServerError)
RespondError(w, r, err.Error(), http.StatusInternalServerError)
return
}
log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", r.ruid)
log.Debug(fmt.Sprintf("Multiple choices! --> %v", list), "ruid", ruid)
//show a nice page links to available entries
ShowMultipleChoices(w, r, list)
return
}
// check the root chunk exists by retrieving the file's size
if _, err := reader.Size(ctx, nil); err != nil {
if _, err := reader.Size(r.Context(), nil); err != nil {
getFileNotFound.Inc(1)
Respond(w, r, fmt.Sprintf("file not found %s: %s", r.uri, err), http.StatusNotFound)
RespondError(w, r, fmt.Sprintf("file not found %s: %s", uri, err), http.StatusNotFound)
return
}
w.Header().Set("Content-Type", contentType)
http.ServeContent(w, &r.Request, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
http.ServeContent(w, r, "", time.Now(), newBufferedReadSeeker(reader, getFileBufferSize))
}
// The size of buffer used for bufio.Reader on LazyChunkReader passed to

299
swarm/api/http/server_test.go
View file

File diff suppressed because one or more lines are too long

357
swarm/api/http/templates.go
View file

File diff suppressed because one or more lines are too long

6
swarm/api/manifest.go
View file

@ -159,7 +159,7 @@ func (m *ManifestWalker) Walk(walkFn WalkFn) error {
}
func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error {
for _, entry := range trie.entries {
for _, entry := range &trie.entries {
if entry == nil {
continue
}
@ -308,7 +308,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) {
}
func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) {
for _, e := range mt.entries {
for _, e := range &mt.entries {
if e != nil {
cnt++
entry = e
@ -362,7 +362,7 @@ func (mt *manifestTrie) recalcAndStore() error {
buffer.WriteString(`{"entries":[`)
list := &Manifest{}
for _, entry := range mt.entries {
for _, entry := range &mt.entries {
if entry != nil {
if entry.Hash == "" { // TODO: paralellize
err := entry.subtrie.recalcAndStore()

2
swarm/bmt/bmt.go
View file

@ -318,7 +318,7 @@ func (h *Hasher) Sum(b []byte) (s []byte) {
// with every full segment calls writeSection in a go routine
func (h *Hasher) Write(b []byte) (int, error) {
l := len(b)
if l == 0 {
if l == 0 || l > 4096 {
return 0, nil
}
t := h.getTree()

13
swarm/network/protocol.go
View file

@ -94,12 +94,14 @@ type BzzConfig struct {
UnderlayAddr []byte // node's underlay address
HiveParams *HiveParams
NetworkID uint64
LightNode bool
}
// Bzz is the swarm protocol bundle
type Bzz struct {
*Hive
NetworkID uint64
LightNode bool
localAddr *BzzAddr
mtx sync.Mutex
handshakes map[discover.NodeID]*HandshakeMsg
@ -116,6 +118,7 @@ func NewBzz(config *BzzConfig, kad Overlay, store state.Store, streamerSpec *pro
return &Bzz{
Hive: NewHive(config.HiveParams, kad, store),
NetworkID: config.NetworkID,
LightNode: config.LightNode,
localAddr: &BzzAddr{config.OverlayAddr, config.UnderlayAddr},
handshakes: make(map[discover.NodeID]*HandshakeMsg),
streamerRun: streamerRun,
@ -209,7 +212,11 @@ func (b *Bzz) RunProtocol(spec *protocols.Spec, run func(*BzzPeer) error) func(*
localAddr: b.localAddr,
BzzAddr: handshake.peerAddr,
lastActive: time.Now(),
LightNode: handshake.LightNode,
}
log.Debug("peer created", "addr", handshake.peerAddr.String())
return run(peer)
}
}
@ -228,6 +235,7 @@ func (b *Bzz) performHandshake(p *protocols.Peer, handshake *HandshakeMsg) error
return err
}
handshake.peerAddr = rsh.(*HandshakeMsg).Addr
handshake.LightNode = rsh.(*HandshakeMsg).LightNode
return nil
}
@ -263,6 +271,7 @@ type BzzPeer struct {
localAddr *BzzAddr // local Peers address
*BzzAddr // remote address -> implements Addr interface = protocols.Peer
lastActive time.Time // time is updated whenever mutexes are releasing
LightNode bool
}
func NewBzzTestPeer(p *protocols.Peer, addr *BzzAddr) *BzzPeer {
@ -294,6 +303,7 @@ type HandshakeMsg struct {
Version uint64
NetworkID uint64
Addr *BzzAddr
LightNode bool
// peerAddr is the address received in the peer handshake
peerAddr *BzzAddr
@ -305,7 +315,7 @@ type HandshakeMsg struct {
// String pretty prints the handshake
func (bh *HandshakeMsg) String() string {
return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v", bh.Version, bh.NetworkID, bh.Addr)
return fmt.Sprintf("Handshake: Version: %v, NetworkID: %v, Addr: %v, LightNode: %v, peerAddr: %v", bh.Version, bh.NetworkID, bh.Addr, bh.LightNode, bh.peerAddr)
}
// Perform initiates the handshake and validates the remote handshake message
@ -338,6 +348,7 @@ func (b *Bzz) GetHandshake(peerID discover.NodeID) (*HandshakeMsg, bool) {
Version: uint64(BzzSpec.Version),
NetworkID: b.NetworkID,
Addr: b.localAddr,
LightNode: b.LightNode,
init: make(chan bool, 1),
done: make(chan struct{}),
}

80
swarm/network/protocol_test.go
View file

@ -30,6 +30,11 @@ import (
p2ptest "github.com/ethereum/go-ethereum/p2p/testing"
)
const (
TestProtocolVersion = 5
TestProtocolNetworkID = 3
)
var (
loglevel = flag.Int("loglevel", 2, "verbosity of logs")
)
@ -127,23 +132,30 @@ type bzzTester struct {
*p2ptest.ProtocolTester
addr *BzzAddr
cs map[string]chan bool
bzz *Bzz
}
func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr) *bzzTester {
func newBzz(addr *BzzAddr, lightNode bool) *Bzz {
config := &BzzConfig{
OverlayAddr: addr.Over(),
UnderlayAddr: addr.Under(),
HiveParams: NewHiveParams(),
NetworkID: DefaultNetworkID,
LightNode: lightNode,
}
kad := NewKademlia(addr.OAddr, NewKadParams())
bzz := NewBzz(config, kad, nil, nil, nil)
return bzz
}
s := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), 1, bzz.runBzz)
func newBzzHandshakeTester(t *testing.T, n int, addr *BzzAddr, lightNode bool) *bzzTester {
bzz := newBzz(addr, lightNode)
pt := p2ptest.NewProtocolTester(t, NewNodeIDFromAddr(addr), n, bzz.runBzz)
return &bzzTester{
addr: addr,
ProtocolTester: s,
ProtocolTester: pt,
bzz: bzz,
}
}
@ -184,22 +196,24 @@ func (s *bzzTester) testHandshake(lhs, rhs *HandshakeMsg, disconnects ...*p2ptes
return nil
}
func correctBzzHandshake(addr *BzzAddr) *HandshakeMsg {
func correctBzzHandshake(addr *BzzAddr, lightNode bool) *HandshakeMsg {
return &HandshakeMsg{
Version: 5,
NetworkID: DefaultNetworkID,
Version: TestProtocolVersion,
NetworkID: TestProtocolNetworkID,
Addr: addr,
LightNode: lightNode,
}
}
func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
lightNode := false
addr := RandomAddr()
s := newBzzHandshakeTester(t, 1, addr)
s := newBzzHandshakeTester(t, 1, addr, lightNode)
id := s.IDs[0]
err := s.testHandshake(
correctBzzHandshake(addr),
&HandshakeMsg{Version: 5, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
correctBzzHandshake(addr, lightNode),
&HandshakeMsg{Version: TestProtocolVersion, NetworkID: 321, Addr: NewAddrFromNodeID(id)},
&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): network id mismatch 321 (!= 3)")},
)
@ -209,14 +223,15 @@ func TestBzzHandshakeNetworkIDMismatch(t *testing.T) {
}
func TestBzzHandshakeVersionMismatch(t *testing.T) {
lightNode := false
addr := RandomAddr()
s := newBzzHandshakeTester(t, 1, addr)
s := newBzzHandshakeTester(t, 1, addr, lightNode)
id := s.IDs[0]
err := s.testHandshake(
correctBzzHandshake(addr),
&HandshakeMsg{Version: 0, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= 5)")},
correctBzzHandshake(addr, lightNode),
&HandshakeMsg{Version: 0, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
&p2ptest.Disconnect{Peer: id, Error: fmt.Errorf("Handshake error: Message handler error: (msg code 0): version mismatch 0 (!= %d)", TestProtocolVersion)},
)
if err != nil {
@ -225,16 +240,49 @@ func TestBzzHandshakeVersionMismatch(t *testing.T) {
}
func TestBzzHandshakeSuccess(t *testing.T) {
lightNode := false
addr := RandomAddr()
s := newBzzHandshakeTester(t, 1, addr)
s := newBzzHandshakeTester(t, 1, addr, lightNode)
id := s.IDs[0]
err := s.testHandshake(
correctBzzHandshake(addr),
&HandshakeMsg{Version: 5, NetworkID: 3, Addr: NewAddrFromNodeID(id)},
correctBzzHandshake(addr, lightNode),
&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: NewAddrFromNodeID(id)},
)
if err != nil {
t.Fatal(err)
}
}
func TestBzzHandshakeLightNode(t *testing.T) {
var lightNodeTests = []struct {
name string
lightNode bool
}{
{"on", true},
{"off", false},
}
for _, test := range lightNodeTests {
t.Run(test.name, func(t *testing.T) {
randomAddr := RandomAddr()
pt := newBzzHandshakeTester(t, 1, randomAddr, false)
id := pt.IDs[0]
addr := NewAddrFromNodeID(id)
err := pt.testHandshake(
correctBzzHandshake(randomAddr, false),
&HandshakeMsg{Version: TestProtocolVersion, NetworkID: TestProtocolNetworkID, Addr: addr, LightNode: test.lightNode},
)
if err != nil {
t.Fatal(err)
}
if pt.bzz.handshakes[id].LightNode != test.lightNode {
t.Fatalf("peer LightNode flag is %v, should be %v", pt.bzz.handshakes[id].LightNode, test.lightNode)
}
})
}
}

7
swarm/sctx/sctx.go Normal file
View file

@ -0,0 +1,7 @@
package sctx
type ContextKey int
const (
HTTPRequestIDKey ContextKey = iota
)

2
swarm/storage/mru/handler.go
View file

@ -469,7 +469,7 @@ func (h *Handler) update(ctx context.Context, r *SignedResourceUpdate) (updateAd
log.Trace("resource update", "updateAddr", r.updateAddr, "lastperiod", r.period, "version", r.version, "data", chunk.SData, "multihash", r.multihash)
// update our resources map entry if the new update is older than the one we have, if we have it.
if rsrc != nil && r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version) {
if rsrc != nil && (r.period > rsrc.period || (rsrc.period == r.period && r.version > rsrc.version)) {
rsrc.period = r.period
rsrc.version = r.version
rsrc.data = make([]byte, len(r.data))

1
swarm/swarm.go
View file

@ -143,6 +143,7 @@ func NewSwarm(config *api.Config, mockStore *mock.NodeStore) (self *Swarm, err e
OverlayAddr: addr.OAddr,
UnderlayAddr: addr.UAddr,
HiveParams: config.HiveParams,
LightNode: config.LightNodeEnabled,
}
stateStore, err := state.NewDBStore(filepath.Join(config.Path, "state-store.db"))

2
trie/database.go
View file

@ -51,7 +51,7 @@ const secureKeyLength = 11 + 32
// DatabaseReader wraps the Get and Has method of a backing store for the trie.
type DatabaseReader interface {
// Get retrieves the value associated with key form the database.
// Get retrieves the value associated with key from the database.
Get(key []byte) (value []byte, err error)
// Has retrieves whether a key is present in the database.

4
trie/node.go
View file

@ -55,7 +55,7 @@ var nilValueNode = valueNode(nil)
func (n *fullNode) EncodeRLP(w io.Writer) error {
var nodes [17]node
for i, child := range n.Children {
for i, child := range &n.Children {
if child != nil {
nodes[i] = child
} else {
@ -98,7 +98,7 @@ func (n valueNode) String() string { return n.fstring("") }
func (n *fullNode) fstring(ind string) string {
resp := fmt.Sprintf("[\n%s ", ind)
for i, node := range n.Children {
for i, node := range &n.Children {
if node == nil {
resp += fmt.Sprintf("%s: <nil> ", indices[i])
} else {

2
trie/trie.go
View file

@ -356,7 +356,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
// value that is left in n or -2 if n contains at least two
// values.
pos := -1
for i, cld := range n.Children {
for i, cld := range &n.Children {
if cld != nil {
if pos == -1 {
pos = i

7
whisper/whisperv6/doc.go
View file

@ -33,7 +33,6 @@ particularly the notion of singular endpoints.
package whisperv6
import (
"fmt"
"time"
)
@ -79,12 +78,6 @@ const (
DefaultSyncAllowance = 10 // seconds
)
type unknownVersionError uint64
func (e unknownVersionError) Error() string {
return fmt.Sprintf("invalid envelope version %d", uint64(e))
}
// MailServer represents a mail server, capable of
// archiving the old messages for subsequent delivery
// to the peers. Any implementation must ensure that both