diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 8803daa56e..0a554bbeb4 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -160,7 +160,7 @@ func (tab *Table) ReadRandomNodes(buf []*Node) (n int) { // Find all non-empty buckets and get a fresh slice of their entries. var buckets [][]*Node - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if len(b.entries) > 0 { buckets = append(buckets, b.entries[:]) } @@ -508,7 +508,7 @@ func (tab *Table) copyLiveNodes() { defer tab.mutex.Unlock() now := time.Now() - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { if now.Sub(n.addedAt) >= seedMinTableTime { tab.db.updateNode(n) @@ -524,7 +524,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { // obviously correct. I believe that tree-based buckets would make // this easier to implement efficiently. close := &nodesByDistance{target: target} - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { close.push(n, nresults) } @@ -533,7 +533,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { } func (tab *Table) len() (n int) { - for _, b := range tab.buckets { + for _, b := range &tab.buckets { n += len(b.entries) } return n diff --git a/p2p/discv5/net_test.go b/p2p/discv5/net_test.go index 001d193cc9..1a8137673d 100644 --- a/p2p/discv5/net_test.go +++ b/p2p/discv5/net_test.go @@ -355,7 +355,7 @@ func (tn *preminedTestnet) mine(target NodeID) { fmt.Printf(" target: %#v,\n", tn.target) fmt.Printf(" targetSha: %#v,\n", tn.targetSha) fmt.Printf(" dists: [%d][]NodeID{\n", len(tn.dists)) - for ld, ns := range tn.dists { + for ld, ns := range &tn.dists { if len(ns) == 0 { continue } diff --git a/p2p/discv5/table.go b/p2p/discv5/table.go index c8d234b936..c793be5082 100644 --- a/p2p/discv5/table.go +++ b/p2p/discv5/table.go @@ -81,7 +81,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash { if printTable { fmt.Println() } - for i, b := range tab.buckets { + for i, b := range &tab.buckets { entries += len(b.entries) if printTable { for _, e := range b.entries { @@ -93,7 +93,7 @@ func (tab *Table) chooseBucketRefreshTarget() common.Hash { prefix := binary.BigEndian.Uint64(tab.self.sha[0:8]) dist := ^uint64(0) entry := int(randUint(uint32(entries + 1))) - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if entry < len(b.entries) { n := b.entries[entry] dist = binary.BigEndian.Uint64(n.sha[0:8]) ^ prefix @@ -121,7 +121,7 @@ func (tab *Table) readRandomNodes(buf []*Node) (n int) { // TODO: tree-based buckets would help here // Find all non-empty buckets and get a fresh slice of their entries. var buckets [][]*Node - for _, b := range tab.buckets { + for _, b := range &tab.buckets { if len(b.entries) > 0 { buckets = append(buckets, b.entries[:]) } @@ -175,7 +175,7 @@ func (tab *Table) closest(target common.Hash, nresults int) *nodesByDistance { // obviously correct. I believe that tree-based buckets would make // this easier to implement efficiently. close := &nodesByDistance{target: target} - for _, b := range tab.buckets { + for _, b := range &tab.buckets { for _, n := range b.entries { close.push(n, nresults) } diff --git a/swarm/README.md b/swarm/README.md index f2d1890434..e81963217f 100644 --- a/swarm/README.md +++ b/swarm/README.md @@ -7,6 +7,21 @@ Swarm is a distributed storage platform and content distribution service, a nati [![Travis](https://travis-ci.org/ethereum/go-ethereum.svg?branch=master)](https://travis-ci.org/ethereum/go-ethereum) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/ethersphere/orange-lounge?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) +## Table of Contents + +* [Building the source](#building-the-source) +* [Running Swarm](#running-swarm) +* [Documentation](#documentation) +* [Developers Guide](#developers-guide) + * [Go Environment](#development-environment) + * [Vendored Dependencies](#vendored-dependencies) + * [Testing](#testing) + * [Profiling Swarm](#profiling-swarm) + * [Metrics and Instrumentation in Swarm](#metrics-and-instrumentation-in-swarm) +* [Public Gateways](#public-gateways) +* [Swarm Dapps](#swarm-dapps) +* [Contributing](#contributing) +* [License](#license) ## Building the source @@ -16,13 +31,187 @@ Building Swarm requires Go (version 1.10 or later). go install github.com/ethereum/go-ethereum/cmd/swarm +## Running Swarm + +Going through all the possible command line flags is out of scope here, but we've enumerated a few common parameter combos to get you up to speed quickly on how you can run your own Swarm node. + +To run Swarm you need an Ethereum account. You can create a new account by running the following command: + + geth account new + +You will be prompted for a password: + + Your new account is locked with a password. Please give a password. Do not forget this password. + Passphrase: + Repeat passphrase: + +Once you have specified the password, the output will be the Ethereum address representing that account. For example: + + Address: {2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1} + +Using this account, connect to Swarm with + + swarm --bzzaccount + + # in our example + + swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 + + +### Verifying that your local Swarm node is running + +When running, Swarm is accessible through an HTTP API on port 8500. + +Confirm that it is up and running by pointing your browser to http://localhost:8500 + +### Ethereum Name Service resolution + +The Ethereum Name Service is the Ethereum equivalent of DNS in the classic web. In order to use ENS to resolve names to Swarm content hashes (e.g. `bzz://theswarm.eth`), `swarm` has to connect to a `geth` instance, which is synced with the Ethereum mainnet. This is done using the `--ens-api` flag. + + swarm --bzzaccount \ + --ens-api '$HOME/.ethereum/geth.ipc' + + # in our example + + swarm --bzzaccount 2f1cd699b0bf461dcfbf0098ad8f5587b038f0f1 \ + --ens-api '$HOME/.ethereum/geth.ipc' + +For more information on usage, features or command line flags, please consult the Documentation. + ## Documentation Swarm documentation can be found at [https://swarm-guide.readthedocs.io](https://swarm-guide.readthedocs.io). -## Contribution +## Developers Guide + +### Go Environment + +We assume that you have Go v1.10 installed, and `GOPATH` is set. + +You must have your working copy under `$GOPATH/src/github.com/ethereum/go-ethereum`. + +Most likely you will be working from your fork of `go-ethereum`, let's say from `github.com/nirname/go-ethereum`. Clone or move your fork into the right place: + +``` +git clone git@github.com:nirname/go-ethereum.git $GOPATH/src/github.com/ethereum/go-ethereum +``` + + +### Vendored Dependencies + +All dependencies are tracked in the `vendor` directory. We use `govendor` to manage them. + +If you want to add a new dependency, run `govendor fetch `, then commit the result. + +If you want to update all dependencies to their latest upstream version, run `govendor fetch +v`. + + +### Testing + +This section explains how to run unit, integration, and end-to-end tests in your development sandbox. + +Testing one library: + +``` +go test -v -cpu 4 ./swarm/api +``` + +Note: Using options -cpu (number of cores allowed) and -v (logging even if no error) is recommended. + +Testing only some methods: + +``` +go test -v -cpu 4 ./eth -run TestMethod +``` + +Note: here all tests with prefix TestMethod will be run, so if you got TestMethod, TestMethod1, then both! + +Running benchmarks: + +``` +go test -v -cpu 4 -bench . -run BenchmarkJoin +``` + + +### Profiling Swarm + +This section explains how to add Go `pprof` profiler to Swarm + +If `swarm` is started with the `--pprof` option, a debugging HTTP server is made available on port 6060. + +You can bring up http://localhost:6060/debug/pprof to see the heap, running routines etc. + +By clicking full goroutine stack dump (clicking http://localhost:6060/debug/pprof/goroutine?debug=2) you can generate trace that is useful for debugging. + + +### Metrics and Instrumentation in Swarm + +This section explains how to visualize and use existing Swarm metrics and how to instrument Swarm with a new metric. + +Swarm metrics system is based on the `go-metrics` library. + +The most common types of measurements we use in Swarm are `counters` and `resetting timers`. Consult the `go-metrics` documentation for full reference of available types. + +``` +# incrementing a counter +metrics.GetOrRegisterCounter("network.stream.received_chunks", nil).Inc(1) + +# measuring latency with a resetting timer +start := time.Now() +t := metrics.GetOrRegisterResettingTimer("http.request.GET.time"), nil) +... +t := UpdateSince(start) +``` + +#### Visualizing metrics + +Swarm supports an InfluxDB exporter. Consult the help section to learn about the command line arguments used to configure it: + +``` +swarm --help | grep metrics +``` + +We use Grafana and InfluxDB to visualise metrics reported by Swarm. We keep our Grafana dashboards under version control at `./swarm/grafana_dashboards`. You could use them or design your own. + +We have built a tool to help with automatic start of Grafana and InfluxDB and provisioning of dashboards at https://github.com/nonsense/stateth , which requires that you have Docker installed. + +Once you have `stateth` installed, and you have Docker running locally, you have to: + +1. Run `stateth` and keep it running in the background +``` +stateth --rm --grafana-dashboards-folder $GOPATH/src/github.com/ethereum/go-ethereum/swarm/grafana_dashboards --influxdb-database metrics +``` + +2. Run `swarm` with at least the following params: +``` +--metrics \ +--metrics.influxdb.export \ +--metrics.influxdb.endpoint "http://localhost:8086" \ +--metrics.influxdb.username "admin" \ +--metrics.influxdb.password "admin" \ +--metrics.influxdb.database "metrics" +``` + +3. Open Grafana at http://localhost:3000 and view the dashboards to gain insight into Swarm. + + +## Public Gateways + +Swarm offers a local HTTP proxy API that Dapps can use to interact with Swarm. The Ethereum Foundation is hosting a public gateway, which allows free access so that people can try Swarm without running their own node. + +The Swarm public gateways are temporary and users should not rely on their existence for production services. + +The Swarm public gateway can be found at https://swarm-gateways.net and is always running the latest `stable` Swarm release. + +## Swarm Dapps + +You can find a few reference Swarm decentralised applications at: https://swarm-gateways.net/bzz:/swarmapps.eth + +Their source code can be found at: https://github.com/ethersphere/swarm-dapps + +## Contributing Thank you for considering to help out with the source code! We welcome contributions from anyone on the internet, and are grateful for even the smallest of fixes! diff --git a/swarm/api/http/response.go b/swarm/api/http/response.go index 32c09b1f57..9f4788d35e 100644 --- a/swarm/api/http/response.go +++ b/swarm/api/http/response.go @@ -29,14 +29,12 @@ import ( "github.com/ethereum/go-ethereum/swarm/api" ) -//metrics variables var ( htmlCounter = metrics.NewRegisteredCounter("api.http.errorpage.html.count", nil) jsonCounter = metrics.NewRegisteredCounter("api.http.errorpage.json.count", nil) plaintextCounter = metrics.NewRegisteredCounter("api.http.errorpage.plaintext.count", nil) ) -//parameters needed for formatting the correct HTML page type ResponseParams struct { Msg template.HTML Code int @@ -45,12 +43,12 @@ type ResponseParams struct { Details template.HTML } -//ShowMultipeChoices is used when a user requests a resource in a manifest which results -//in ambiguous results. It returns a HTML page with clickable links of each of the entry -//in the manifest which fits the request URI ambiguity. -//For example, if the user requests bzz://read and that manifest contains entries -//"readme.md" and "readinglist.txt", a HTML page is returned with this two links. -//This only applies if the manifest has no default entry +// ShowMultipleChoices is used when a user requests a resource in a manifest which results +// in ambiguous results. It returns a HTML page with clickable links of each of the entry +// in the manifest which fits the request URI ambiguity. +// For example, if the user requests bzz://read and that manifest contains entries +// "readme.md" and "readinglist.txt", a HTML page is returned with this two links. +// This only applies if the manifest has no default entry func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.ManifestList) { log.Debug("ShowMultipleChoices", "ruid", GetRUID(r.Context()), "uri", GetURI(r.Context())) msg := "" @@ -66,7 +64,6 @@ func ShowMultipleChoices(w http.ResponseWriter, r *http.Request, list api.Manife } uri.Scheme = "bzz-list" - //request the same url just with bzz-list msg += fmt.Sprintf("Disambiguation:
Your request may refer to multiple choices.
Click here if your browser does not redirect you within 5 seconds.
", "/"+uri.String()) RespondTemplate(w, r, "error", msg, http.StatusMultipleChoices) } @@ -86,7 +83,6 @@ func RespondError(w http.ResponseWriter, r *http.Request, msg string, code int) RespondTemplate(w, r, "error", msg, code) } -//evaluate if client accepts html or json response func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { w.WriteHeader(params.Code) @@ -108,7 +104,6 @@ func respond(w http.ResponseWriter, r *http.Request, params *ResponseParams) { } } -//return a HTML page func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) { htmlCounter.Inc(1) log.Debug("respondHTML", "ruid", GetRUID(r.Context())) @@ -118,7 +113,6 @@ func respondHTML(w http.ResponseWriter, r *http.Request, params *ResponseParams) } } -//return JSON func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { jsonCounter.Inc(1) log.Debug("respondJSON", "ruid", GetRUID(r.Context())) @@ -126,7 +120,6 @@ func respondJSON(w http.ResponseWriter, r *http.Request, params *ResponseParams) return json.NewEncoder(w).Encode(params) } -//return plaintext func respondPlaintext(w http.ResponseWriter, r *http.Request, params *ResponseParams) error { plaintextCounter.Inc(1) log.Debug("respondPlaintext", "ruid", GetRUID(r.Context())) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index f23f236b22..3ac60596b5 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -909,7 +909,6 @@ func TestMethodsNotAllowed(t *testing.T) { } -// HTTP convenience function func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string]string, verbose bool, t *testing.T) (*http.Response, string) { // Build the Request req, err := http.NewRequest(httpMethod, url, reqBody) @@ -942,11 +941,10 @@ func httpDo(httpMethod string, url string, reqBody io.Reader, headers map[string } func TestGet(t *testing.T) { - // Setup Swarm srv := testutil.NewTestSwarmServer(t, serverFunc) defer srv.Close() - testCases := []struct { + for _, testCase := range []struct { uri string method string headers map[string]string @@ -955,25 +953,22 @@ func TestGet(t *testing.T) { verbose bool }{ { - // Accept: text/html GET / -> 200 HTML, Swarm Landing Page uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, expectedStatusCode: 200, - assertResponseBody: "Swarm: Serverless Hosting Incentivised peer-to-peer Storage and Content Distribution", + assertResponseBody: "Swarm: Serverless Hosting Incentivised Peer-To-Peer Storage And Content Distribution", verbose: false, }, { - // Accept: application/json GET / -> 200 'Welcome to Swarm' uri: fmt.Sprintf("%s/", srv.URL), method: "GET", headers: map[string]string{"Accept": "application/json"}, expectedStatusCode: 200, - assertResponseBody: "Welcome to Swarm!", + assertResponseBody: "Swarm: Please request a valid ENS or swarm hash with the appropriate bzz scheme", verbose: false, }, { - // GET /robots.txt -> 200 uri: fmt.Sprintf("%s/robots.txt", srv.URL), method: "GET", headers: map[string]string{"Accept": "text/html"}, @@ -982,62 +977,54 @@ func TestGet(t *testing.T) { verbose: false, }, { - // GET /path_that_doesnt exist -> 400 uri: fmt.Sprintf("%s/nonexistent_path", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 400, + expectedStatusCode: 404, verbose: false, }, { - // GET bzz-invalid:/ -> 400 uri: fmt.Sprintf("%s/bzz:asdf/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 400, + expectedStatusCode: 404, verbose: false, }, { - // GET bzz-invalid:/ -> 400 uri: fmt.Sprintf("%s/tbz2/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 400, + expectedStatusCode: 404, verbose: false, }, { - // GET bzz-invalid:/ -> 400 uri: fmt.Sprintf("%s/bzz-rack:/", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 400, + expectedStatusCode: 404, verbose: false, }, { - // GET bzz-invalid:/ -> 400 uri: fmt.Sprintf("%s/bzz-ls", srv.URL), method: "GET", headers: map[string]string{}, - expectedStatusCode: 400, + expectedStatusCode: 404, verbose: false, }, - } - - for _, testCase := range testCases { + } { t.Run("GET "+testCase.uri, func(t *testing.T) { res, body := httpDo(testCase.method, testCase.uri, nil, testCase.headers, testCase.verbose, t) if res.StatusCode != testCase.expectedStatusCode { - t.Fatalf("expected %s %s to return a %v but it didn't", testCase.method, testCase.uri, testCase.expectedStatusCode) + t.Fatalf("expected status code %d but got %d", testCase.expectedStatusCode, res.StatusCode) } if testCase.assertResponseBody != "" && !strings.Contains(body, testCase.assertResponseBody) { - t.Fatalf("expected %s %s to have %s within HTTP response body but it didn't", testCase.method, testCase.uri, testCase.assertResponseBody) + t.Fatalf("expected response to be: %s but got: %s", testCase.assertResponseBody, body) } }) } } func TestModify(t *testing.T) { - // Setup Swarm and upload a test file to it srv := testutil.NewTestSwarmServer(t, serverFunc) defer srv.Close() @@ -1057,7 +1044,7 @@ func TestModify(t *testing.T) { t.Fatal(err) } - testCases := []struct { + for _, testCase := range []struct { uri string method string headers map[string]string @@ -1068,7 +1055,6 @@ func TestModify(t *testing.T) { verbose bool }{ { - // DELETE bzz:/hash -> 200 OK uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "DELETE", headers: map[string]string{}, @@ -1077,7 +1063,6 @@ func TestModify(t *testing.T) { verbose: false, }, { - // PUT bzz:/hash -> 405 Method Not Allowed uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, @@ -1085,7 +1070,6 @@ func TestModify(t *testing.T) { verbose: false, }, { - // PUT bzz-raw:/hash -> 405 Method Not Allowed uri: fmt.Sprintf("%s/bzz-raw:/%s", srv.URL, hash), method: "PUT", headers: map[string]string{}, @@ -1093,7 +1077,6 @@ func TestModify(t *testing.T) { verbose: false, }, { - // PATCH bzz:/hash -> 405 Method Not Allowed uri: fmt.Sprintf("%s/bzz:/%s", srv.URL, hash), method: "PATCH", headers: map[string]string{}, @@ -1101,7 +1084,6 @@ func TestModify(t *testing.T) { verbose: false, }, { - // POST bzz-raw:/ -> 200 OK uri: fmt.Sprintf("%s/bzz-raw:/", srv.URL), method: "POST", headers: map[string]string{}, @@ -1111,7 +1093,6 @@ func TestModify(t *testing.T) { verbose: false, }, { - // POST bzz-raw:/encrypt -> 200 OK uri: fmt.Sprintf("%s/bzz-raw:/encrypt", srv.URL), method: "POST", headers: map[string]string{}, @@ -1120,19 +1101,17 @@ func TestModify(t *testing.T) { assertResponseHeaders: map[string]string{"Content-Length": "128"}, verbose: false, }, - } - - for _, testCase := range testCases { + } { t.Run(testCase.method+" "+testCase.uri, func(t *testing.T) { reqBody := bytes.NewReader(testCase.requestBody) res, body := httpDo(testCase.method, testCase.uri, reqBody, testCase.headers, testCase.verbose, t) if res.StatusCode != testCase.expectedStatusCode { - t.Fatalf("expected %s %s to return a %v but it returned a %v instead", testCase.method, testCase.uri, testCase.expectedStatusCode, res.StatusCode) + t.Fatalf("expected status code %d but got %d", testCase.expectedStatusCode, res.StatusCode) } if testCase.assertResponseBody != "" && !strings.Contains(body, testCase.assertResponseBody) { t.Log(body) - t.Fatalf("expected %s %s to have %s within HTTP response body but it didn't", testCase.method, testCase.uri, testCase.assertResponseBody) + t.Fatalf("expected response %s but got %s", testCase.assertResponseBody, body) } for key, value := range testCase.assertResponseHeaders { if res.Header.Get(key) != value { diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index 198ca22ce0..fbd143f295 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -159,7 +159,7 @@ func (m *ManifestWalker) Walk(walkFn WalkFn) error { } func (m *ManifestWalker) walk(trie *manifestTrie, prefix string, walkFn WalkFn) error { - for _, entry := range trie.entries { + for _, entry := range &trie.entries { if entry == nil { continue } @@ -308,7 +308,7 @@ func (mt *manifestTrie) addEntry(entry *manifestTrieEntry, quitC chan bool) { } func (mt *manifestTrie) getCountLast() (cnt int, entry *manifestTrieEntry) { - for _, e := range mt.entries { + for _, e := range &mt.entries { if e != nil { cnt++ entry = e @@ -362,7 +362,7 @@ func (mt *manifestTrie) recalcAndStore() error { buffer.WriteString(`{"entries":[`) list := &Manifest{} - for _, entry := range mt.entries { + for _, entry := range &mt.entries { if entry != nil { if entry.Hash == "" { // TODO: paralellize err := entry.subtrie.recalcAndStore() diff --git a/trie/node.go b/trie/node.go index a06f1b3898..1fafb7a538 100644 --- a/trie/node.go +++ b/trie/node.go @@ -55,7 +55,7 @@ var nilValueNode = valueNode(nil) func (n *fullNode) EncodeRLP(w io.Writer) error { var nodes [17]node - for i, child := range n.Children { + for i, child := range &n.Children { if child != nil { nodes[i] = child } else { @@ -98,7 +98,7 @@ func (n valueNode) String() string { return n.fstring("") } func (n *fullNode) fstring(ind string) string { resp := fmt.Sprintf("[\n%s ", ind) - for i, node := range n.Children { + for i, node := range &n.Children { if node == nil { resp += fmt.Sprintf("%s: ", indices[i]) } else { diff --git a/trie/trie.go b/trie/trie.go index 4284e30ad4..e920ccd23f 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -356,7 +356,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // value that is left in n or -2 if n contains at least two // values. pos := -1 - for i, cld := range n.Children { + for i, cld := range &n.Children { if cld != nil { if pos == -1 { pos = i