From b520d7caa5f8a3307559db245974f0243b7908fd Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 22 Dec 2016 00:54:29 +0100 Subject: [PATCH 01/12] Command line now overrides saved config, saved config overrides system default --- cmd/swarm/main.go | 8 ++++++-- swarm/api/config.go | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 04930760ee..3551a59da4 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/network" + //"github.com/ethereum/go-ethereum/swarm/network" "gopkg.in/urfave/cli.v1" ) @@ -76,7 +76,8 @@ var ( SwarmNetworkIdFlag = cli.IntFlag{ Name: "bzznetworkid", Usage: "Network identifier (integer, default 3=swarm testnet)", - Value: network.NetworkId, + // defining this here means we can't distinguish if it was passed at cmd line or system default when deciding to use saved values from config + //Value: network.NetworkId, } SwarmConfigPathFlag = cli.StringFlag{ Name: "bzzconfig", @@ -237,6 +238,7 @@ func bzzd(ctx *cli.Context) error { } func registerBzzService(ctx *cli.Context, stack *node.Node) { + prvkey := getAccount(ctx, stack) chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name)) @@ -244,6 +246,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { if bzzdir == "" { bzzdir = stack.InstanceDir() } + bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name)) if err != nil { utils.Fatalf("unable to configure swarm: %v", err) @@ -274,6 +277,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { keyid := ctx.GlobalString(SwarmAccountFlag.Name) + if keyid == "" { utils.Fatalf("Option %q is required", SwarmAccountFlag.Name) } diff --git a/swarm/api/config.go b/swarm/api/config.go index 14a559c75d..72822f3d98 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -70,7 +70,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) pubkeyhex := common.ToHex(pubkey) keyhex := crypto.Sha3Hash(pubkey).Hex() - + self = &Config{ SyncParams: network.NewSyncParams(dirpath), HiveParams: network.NewHiveParams(dirpath), @@ -85,10 +85,16 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n NetworkId: networkId, } data, err = ioutil.ReadFile(confpath) + + if networkId == 0 { + self.NetworkId = network.NetworkId + } + if err != nil { if !os.IsNotExist(err) { return } + // file does not exist // write out config file err = self.Save() @@ -97,6 +103,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n } return } + // file exists, deserialise err = json.Unmarshal(data, self) if err != nil { @@ -109,8 +116,13 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n if keyhex != self.BzzKey { return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) } + + if networkId != 0 { + self.NetworkId = networkId + } + self.Swap.SetKey(prvKey) - + if (self.EnsRoot == common.Address{}) { self.EnsRoot = ensRootAddress } From 5fb1c2a6012ea687a6d2c72faf8a2b4298cd9e32 Mon Sep 17 00:00:00 2001 From: nolash Date: Sat, 24 Dec 2016 13:07:44 +0100 Subject: [PATCH 02/12] swarm: read format and comment docs, updated fixes ethereum/go-ethereum#3444 --- cmd/swarm/main.go | 8 +++----- swarm/api/config.go | 14 +++++++------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 3551a59da4..fc6e9127d3 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -76,8 +76,6 @@ var ( SwarmNetworkIdFlag = cli.IntFlag{ Name: "bzznetworkid", Usage: "Network identifier (integer, default 3=swarm testnet)", - // defining this here means we can't distinguish if it was passed at cmd line or system default when deciding to use saved values from config - //Value: network.NetworkId, } SwarmConfigPathFlag = cli.StringFlag{ Name: "bzzconfig", @@ -238,7 +236,7 @@ func bzzd(ctx *cli.Context) error { } func registerBzzService(ctx *cli.Context, stack *node.Node) { - + prvkey := getAccount(ctx, stack) chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name)) @@ -246,7 +244,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { if bzzdir == "" { bzzdir = stack.InstanceDir() } - + bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name)) if err != nil { utils.Fatalf("unable to configure swarm: %v", err) @@ -277,7 +275,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { keyid := ctx.GlobalString(SwarmAccountFlag.Name) - + if keyid == "" { utils.Fatalf("Option %q is required", SwarmAccountFlag.Name) } diff --git a/swarm/api/config.go b/swarm/api/config.go index 72822f3d98..acb861cbfe 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -70,7 +70,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n pubkey := crypto.FromECDSAPub(&prvKey.PublicKey) pubkeyhex := common.ToHex(pubkey) keyhex := crypto.Sha3Hash(pubkey).Hex() - + self = &Config{ SyncParams: network.NewSyncParams(dirpath), HiveParams: network.NewHiveParams(dirpath), @@ -89,12 +89,12 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n if networkId == 0 { self.NetworkId = network.NetworkId } - + if err != nil { if !os.IsNotExist(err) { return } - + // file does not exist // write out config file err = self.Save() @@ -103,7 +103,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n } return } - + // file exists, deserialise err = json.Unmarshal(data, self) if err != nil { @@ -116,13 +116,13 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n if keyhex != self.BzzKey { return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) } - + if networkId != 0 { self.NetworkId = networkId } - + self.Swap.SetKey(prvKey) - + if (self.EnsRoot == common.Address{}) { self.EnsRoot = ensRootAddress } From 922f1055a4843443b3be2b392a6588033b1f5581 Mon Sep 17 00:00:00 2001 From: nolash Date: Mon, 2 Jan 2017 14:27:40 +0100 Subject: [PATCH 03/12] Parse url before resolve when path and ENS is supplied, example for illustration --- swarm/api/http/server.go | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index f82775f254..8b1bf21c6e 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" ) const ( @@ -175,16 +176,40 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { case r.Method == "GET" || r.Method == "HEAD": path = trailingSlashes.ReplaceAllString(path, "") if raw { + var reader storage.LazySectionReader + // + parsedurl, _ := api.Parse(path) + //if err != nil { + // + //} + // resolving host - key, err := a.Resolve(path, nameresolver) - if err != nil { - glog.V(logger.Error).Infof("%v", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return + if (parsedurl == path) { + key, err := a.Resolve(parsedurl, nameresolver) + if err != nil { + glog.V(logger.Error).Infof("%v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + reader = a.Retrieve(key) + } else { + readertmp, _, status, err := a.Get(path, nameresolver) + reader = readertmp + if err != nil { + if _, ok := err.(api.ErrResolve); ok { + glog.V(logger.Debug).Infof("%v", err) + status = http.StatusBadRequest + } else { + glog.V(logger.Debug).Infof("error retrieving '%s': %v", uri, err) + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) + return + } } - + // retrieving content - reader := a.Retrieve(key) + quitC := make(chan bool) size, err := reader.Size(quitC) if err != nil { From 90daa7a68b0568ddf8a65fb644cdaddd8254a7b6 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Jan 2017 22:04:08 +0100 Subject: [PATCH 04/12] swarm/api/http proxy server test for retrieval of subpath through get Removed nil entry assignment on subtrie leaf in recursive key retrieval Cleaned up path-or-no-path condition in proxy server get handler --- swarm/api/http/server.go | 23 ++----- swarm/api/http/server_test.go | 111 ++++++++++++++++++++++++++++++++++ swarm/api/manifest.go | 7 ++- 3 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 swarm/api/http/server_test.go diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 8b1bf21c6e..4ec15e7693 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -177,14 +177,9 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { path = trailingSlashes.ReplaceAllString(path, "") if raw { var reader storage.LazySectionReader - // parsedurl, _ := api.Parse(path) - //if err != nil { - // - //} - - // resolving host - if (parsedurl == path) { + + if parsedurl == path { key, err := a.Resolve(parsedurl, nameresolver) if err != nil { glog.V(logger.Error).Infof("%v", err) @@ -193,21 +188,15 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } reader = a.Retrieve(key) } else { + var status int readertmp, _, status, err := a.Get(path, nameresolver) - reader = readertmp if err != nil { - if _, ok := err.(api.ErrResolve); ok { - glog.V(logger.Debug).Infof("%v", err) - status = http.StatusBadRequest - } else { - glog.V(logger.Debug).Infof("error retrieving '%s': %v", uri, err) - status = http.StatusNotFound - } http.Error(w, err.Error(), status) return } + reader = readertmp } - + // retrieving content quitC := make(chan bool) @@ -230,7 +219,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { w.Header().Set("Content-Type", mimeType) http.ServeContent(w, r, uri, forever(), reader) glog.V(logger.Debug).Infof("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType) - + // retrieve path via manifest } else { glog.V(logger.Debug).Infof("Structured GET request '%s' received.", uri) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go new file mode 100644 index 0000000000..c3ac94c03c --- /dev/null +++ b/swarm/api/http/server_test.go @@ -0,0 +1,111 @@ +package http + +import ( + "testing" + "io/ioutil" + "net/http" + "bytes" + "sync" + "time" + + "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger/glog" +) + +func TestBzzrGetPath(t *testing.T) { + + glog.SetToStderr(true) + glog.SetV(6) + + var err error + + maxproxyattempts := 3 + + testmanifest := []string{ + `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, + `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, + `{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`, + } + + testrequests := make(map[string]int) + testrequests["/"] = 0 + testrequests["/a"] = 1 + testrequests["/a/b"] = 2 + + reader := [3]*bytes.Reader{} + + key := [3]storage.Key{} + + dir, _ := ioutil.TempDir("", "bzz-storage-test") + + storeparams := &storage.StoreParams{ + dir, + 5000000, + 5000, + 0, + } + + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + if err != nil { + t.Fatal(err) + } + chunker := storage.NewTreeChunker(storage.NewChunkerParams()) + dpa := &storage.DPA{ + Chunker: chunker, + ChunkStore: localStore, + } + dpa.Start() + defer dpa.Stop() + + wg := &sync.WaitGroup{} + + for i, mf := range testmanifest { + reader[i] = bytes.NewReader([]byte(mf)) + key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) + if err != nil { + t.Fatal(err) + } + wg.Wait() + } + + a := api.NewApi(dpa, nil) + + // iterate port numbers up if fail + StartHttpServer(a, "8504") + // how to wait for ListenAndServe to have initialized? This is pretty cruuuude + // if we fix it we don't need maxproxyattempts anymore either + time.Sleep(100 * time.Millisecond) + for i := 0; i <= maxproxyattempts; i++ { + _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") + if i == maxproxyattempts { + t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err) + } else if err != nil { + t.Logf("Proxy connect failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + break + } + + + + for k, v := range testrequests { + var body []byte + var resp *http.Response + url := "http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" + t.Logf("Sending proxy GET: %v", url) + resp, err = http.Get(url) + defer resp.Body.Close() + body, err = ioutil.ReadAll(resp.Body) + + if string(body) != testmanifest[v] { + t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(body)) + } + + t.Log(string(body)) + } + + +} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d6dc24c486..a5076f0a7f 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -302,7 +302,8 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man if (len(path) >= epl) && (path[:epl] == entry.Path) { glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType) if entry.ContentType == manifestType { - if self.loadSubTrie(entry, quitC) != nil { + err := self.loadSubTrie(entry, quitC) + if err != nil { return nil, 0 } entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) @@ -312,9 +313,9 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man } else { pos = epl } - } else { + } /*else { entry = nil - } + }*/ return } From 5cb586c8acde851f518a607174c4e55e7f22fd1f Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Jan 2017 22:10:40 +0100 Subject: [PATCH 05/12] swarm: processed with gofmt refers to lash/go-ethereum@90daa7a68b0568ddf8a65fb644cdaddd8254a7b6 --- swarm/api/http/server.go | 2 +- swarm/api/http/server_test.go | 53 +++++++++++++++++------------------ 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 4ec15e7693..83c879aec3 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -219,7 +219,7 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { w.Header().Set("Content-Type", mimeType) http.ServeContent(w, r, uri, forever(), reader) glog.V(logger.Debug).Infof("Serve raw content '%s' (%d bytes) as '%s'", uri, size, mimeType) - + // retrieve path via manifest } else { glog.V(logger.Debug).Infof("Structured GET request '%s' received.", uri) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index c3ac94c03c..2d088dfd5c 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -1,45 +1,45 @@ package http import ( - "testing" + "bytes" "io/ioutil" "net/http" - "bytes" "sync" + "testing" "time" - - "github.com/ethereum/go-ethereum/swarm/storage" - "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" ) func TestBzzrGetPath(t *testing.T) { - + glog.SetToStderr(true) glog.SetV(6) - + var err error - + maxproxyattempts := 3 - + testmanifest := []string{ `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, `{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`, } - + testrequests := make(map[string]int) testrequests["/"] = 0 testrequests["/a"] = 1 testrequests["/a/b"] = 2 - + reader := [3]*bytes.Reader{} - - key := [3]storage.Key{} - + + key := [3]storage.Key{} + dir, _ := ioutil.TempDir("", "bzz-storage-test") - + storeparams := &storage.StoreParams{ dir, 5000000, @@ -58,9 +58,9 @@ func TestBzzrGetPath(t *testing.T) { } dpa.Start() defer dpa.Stop() - + wg := &sync.WaitGroup{} - + for i, mf := range testmanifest { reader[i] = bytes.NewReader([]byte(mf)) key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) @@ -69,28 +69,26 @@ func TestBzzrGetPath(t *testing.T) { } wg.Wait() } - + a := api.NewApi(dpa, nil) - + // iterate port numbers up if fail StartHttpServer(a, "8504") // how to wait for ListenAndServe to have initialized? This is pretty cruuuude // if we fix it we don't need maxproxyattempts anymore either - time.Sleep(100 * time.Millisecond) + time.Sleep(100 * time.Millisecond) for i := 0; i <= maxproxyattempts; i++ { _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") if i == maxproxyattempts { t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err) } else if err != nil { t.Logf("Proxy connect failed: %v", err) - time.Sleep(100 * time.Millisecond) + time.Sleep(100 * time.Millisecond) continue - } + } break } - - - + for k, v := range testrequests { var body []byte var resp *http.Response @@ -99,13 +97,12 @@ func TestBzzrGetPath(t *testing.T) { resp, err = http.Get(url) defer resp.Body.Close() body, err = ioutil.ReadAll(resp.Body) - + if string(body) != testmanifest[v] { t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(body)) } - + t.Log(string(body)) } - } From 2ec3fd7d00cea5e4bd9079bb95d086c1d390c719 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Jan 2017 22:41:56 +0100 Subject: [PATCH 06/12] swarm: Added public access method Parse alias to parse --- swarm/api/api.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/swarm/api/api.go b/swarm/api/api.go index 673cff350a..63de449b08 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -22,6 +22,7 @@ import ( "regexp" "strings" "sync" + "net/http" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/logger" @@ -71,6 +72,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { + glog.V(logger.Warn).Infof("Resolving : %v", hostPort) if hashMatcher.MatchString(hostPort) || self.dns == nil { glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort) return storage.Key(common.Hex2Bytes(hostPort)), nil @@ -87,6 +89,10 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error return contentHash[:], err } +func Parse(url string) (hostPost, path string) { + return parse(url) +} + func parse(uri string) (hostPort, path string) { parts := slashes.Split(uri, 3) var i int @@ -150,7 +156,9 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR } glog.V(logger.Detail).Infof("getEntry(%s)", path) + entry, _ := trie.getEntry(path) + if entry != nil { key = common.Hex2Bytes(entry.Hash) status = entry.Status @@ -158,6 +166,7 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType) reader = self.dpa.Retrieve(key) } else { + status = http.StatusNotFound err = fmt.Errorf("manifest entry for '%s' not found", path) glog.V(logger.Warn).Infof("%v", err) } From e1dde04eb27cc22c5b2b5ad9c8cbf6816cdbabf5 Mon Sep 17 00:00:00 2001 From: nolash Date: Tue, 3 Jan 2017 22:44:38 +0100 Subject: [PATCH 07/12] swarm: processed with gofmt References nolash/go-ethereum@2ec3fd7d00cea5e4bd9079bb95d086c1d390c719 --- swarm/api/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 63de449b08..76754ecb63 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -19,10 +19,10 @@ package api import ( "fmt" "io" + "net/http" "regexp" "strings" "sync" - "net/http" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/logger" @@ -156,9 +156,9 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR } glog.V(logger.Detail).Infof("getEntry(%s)", path) - + entry, _ := trie.getEntry(path) - + if entry != nil { key = common.Hex2Bytes(entry.Hash) status = entry.Status From ea88565fad275cc9520f6d6fb0a3c5f4b3f654df Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Jan 2017 16:11:10 +0100 Subject: [PATCH 08/12] Rename parse to Parse, removed alias --- swarm/api/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 76754ecb63..a10b025acd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -72,7 +72,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { - glog.V(logger.Warn).Infof("Resolving : %v", hostPort) + glog.V(logger.Detail).Infof("Resolving : %v", hostPort) if hashMatcher.MatchString(hostPort) || self.dns == nil { glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort) return storage.Key(common.Hex2Bytes(hostPort)), nil @@ -89,11 +89,11 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error return contentHash[:], err } -func Parse(url string) (hostPost, path string) { +/*func Parse(url string) (hostPost, path string) { return parse(url) -} +}*/ -func parse(uri string) (hostPort, path string) { +func Parse(uri string) (hostPort, path string) { parts := slashes.Split(uri, 3) var i int if len(parts) == 0 { From 8a96f1e8807cfbd8c8e798a311a30a8c1d1b4f1f Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Jan 2017 16:11:10 +0100 Subject: [PATCH 09/12] Rename parse to Parse, removed alias --- swarm/api/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 76754ecb63..a10b025acd 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -72,7 +72,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { - glog.V(logger.Warn).Infof("Resolving : %v", hostPort) + glog.V(logger.Detail).Infof("Resolving : %v", hostPort) if hashMatcher.MatchString(hostPort) || self.dns == nil { glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort) return storage.Key(common.Hex2Bytes(hostPort)), nil @@ -89,11 +89,11 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error return contentHash[:], err } -func Parse(url string) (hostPost, path string) { +/*func Parse(url string) (hostPost, path string) { return parse(url) -} +}*/ -func parse(uri string) (hostPort, path string) { +func Parse(uri string) (hostPort, path string) { parts := slashes.Split(uri, 3) var i int if len(parts) == 0 { From e371cfbf4aa02f95834bbfd392cc8dbe30b77cac Mon Sep 17 00:00:00 2001 From: nolash Date: Thu, 22 Dec 2016 00:54:29 +0100 Subject: [PATCH 10/12] swarm/network,swarm/api: networkid override, bzzr handle path fixes ethereum/go-ethereum#3444 * Command line now overrides saved config, saved config overrides system default * swarm: read format and comment docs, updated * Parse url before resolve when path and ENS is supplied, example for illustration * swarm/api/http proxy server test for retrieval of subpath through get * Removed nil entry assignment on subtrie leaf in recursive key retrieval * Cleaned up path-or-no-path condition in proxy server get handler * swarm: processed with gofmt refers to lash/go-ethereum@90daa7a68b0568ddf8a65fb644cdaddd8254a7b6 * swarm: Added public access method Parse alias to parse * swarm: processed with gofmt References nolash/go-ethereum@2ec3fd7d00cea5e4bd9079bb95d086c1d390c719 * Rename parse to Parse, removed alias * swarm: gofmt --- cmd/swarm/main.go | 6 +- swarm/api/api.go | 10 +++- swarm/api/config.go | 12 ++++ swarm/api/http/server.go | 28 ++++++--- swarm/api/http/server_test.go | 108 ++++++++++++++++++++++++++++++++++ swarm/api/manifest.go | 7 ++- 6 files changed, 156 insertions(+), 15 deletions(-) create mode 100644 swarm/api/http/server_test.go diff --git a/cmd/swarm/main.go b/cmd/swarm/main.go index 87e21fb7fc..0cda1e5018 100644 --- a/cmd/swarm/main.go +++ b/cmd/swarm/main.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" "github.com/ethereum/go-ethereum/swarm" bzzapi "github.com/ethereum/go-ethereum/swarm/api" - "github.com/ethereum/go-ethereum/swarm/network" + //"github.com/ethereum/go-ethereum/swarm/network" "gopkg.in/urfave/cli.v1" ) @@ -76,7 +76,6 @@ var ( SwarmNetworkIdFlag = cli.IntFlag{ Name: "bzznetworkid", Usage: "Network identifier (integer, default 3=swarm testnet)", - Value: network.NetworkId, } SwarmConfigPathFlag = cli.StringFlag{ Name: "bzzconfig", @@ -242,6 +241,7 @@ func bzzd(ctx *cli.Context) error { } func registerBzzService(ctx *cli.Context, stack *node.Node) { + prvkey := getAccount(ctx, stack) chbookaddr := common.HexToAddress(ctx.GlobalString(ChequebookAddrFlag.Name)) @@ -249,6 +249,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { if bzzdir == "" { bzzdir = stack.InstanceDir() } + bzzconfig, err := bzzapi.NewConfig(bzzdir, chbookaddr, prvkey, ctx.GlobalUint64(SwarmNetworkIdFlag.Name)) if err != nil { utils.Fatalf("unable to configure swarm: %v", err) @@ -280,6 +281,7 @@ func registerBzzService(ctx *cli.Context, stack *node.Node) { func getAccount(ctx *cli.Context, stack *node.Node) *ecdsa.PrivateKey { keyid := ctx.GlobalString(SwarmAccountFlag.Name) + if keyid == "" { utils.Fatalf("Option %q is required", SwarmAccountFlag.Name) } diff --git a/swarm/api/api.go b/swarm/api/api.go index 3f48437a5c..10f2597f39 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -19,6 +19,7 @@ package api import ( "fmt" "io" + "net/http" "regexp" "strings" "sync" @@ -71,6 +72,7 @@ type ErrResolve error // DNS Resolver func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error) { + glog.V(logger.Detail).Infof("Resolving : %v", hostPort) if hashMatcher.MatchString(hostPort) || self.dns == nil { glog.V(logger.Detail).Infof("host is a contentHash: '%v'", hostPort) return storage.Key(common.Hex2Bytes(hostPort)), nil @@ -86,8 +88,7 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error glog.V(logger.Detail).Infof("host lookup: %v -> %v", err) return contentHash[:], err } - -func parse(uri string) (hostPort, path string) { +func Parse(uri string) (hostPort, path string) { parts := slashes.Split(uri, 3) var i int if len(parts) == 0 { @@ -111,7 +112,7 @@ func parse(uri string) (hostPort, path string) { } func (self *Api) parseAndResolve(uri string, nameresolver bool) (key storage.Key, hostPort, path string, err error) { - hostPort, path = parse(uri) + hostPort, path = Parse(uri) //resolving host and port contentHash, err := self.Resolve(hostPort, nameresolver) glog.V(logger.Debug).Infof("Resolved '%s' to contentHash: '%s', path: '%s'", uri, contentHash, path) @@ -153,7 +154,9 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR } glog.V(logger.Detail).Infof("getEntry(%s)", path) + entry, _ := trie.getEntry(path) + if entry != nil { key = common.Hex2Bytes(entry.Hash) status = entry.Status @@ -161,6 +164,7 @@ func (self *Api) Get(uri string, nameresolver bool) (reader storage.LazySectionR glog.V(logger.Detail).Infof("content lookup key: '%v' (%v)", key, mimeType) reader = self.dpa.Retrieve(key) } else { + status = http.StatusNotFound err = fmt.Errorf("manifest entry for '%s' not found", path) glog.V(logger.Warn).Infof("%v", err) } diff --git a/swarm/api/config.go b/swarm/api/config.go index b4c6e3d4a3..e4c2dfb5fc 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -85,10 +85,16 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n NetworkId: networkId, } data, err = ioutil.ReadFile(confpath) + + if networkId == 0 { + self.NetworkId = network.NetworkId + } + if err != nil { if !os.IsNotExist(err) { return } + // file does not exist // write out config file err = self.Save() @@ -97,6 +103,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n } return } + // file exists, deserialise err = json.Unmarshal(data, self) if err != nil { @@ -109,6 +116,11 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n if keyhex != self.BzzKey { return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) } + + if networkId != 0 { + self.NetworkId = networkId + } + self.Swap.SetKey(prvKey) if (self.EnsRoot == common.Address{}) { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index c8e79ab4ec..85505776d0 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/api" "github.com/rs/cors" + "github.com/ethereum/go-ethereum/swarm/storage" ) const ( @@ -195,16 +196,29 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { case r.Method == "GET" || r.Method == "HEAD": path = trailingSlashes.ReplaceAllString(path, "") if raw { - // resolving host - key, err := a.Resolve(path, nameresolver) - if err != nil { - glog.V(logger.Error).Infof("%v", err) - http.Error(w, err.Error(), http.StatusBadRequest) - return + var reader storage.LazySectionReader + parsedurl, _ := api.Parse(path) + + if parsedurl == path { + key, err := a.Resolve(parsedurl, nameresolver) + if err != nil { + glog.V(logger.Error).Infof("%v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + reader = a.Retrieve(key) + } else { + var status int + readertmp, _, status, err := a.Get(path, nameresolver) + if err != nil { + http.Error(w, err.Error(), status) + return + } + reader = readertmp } // retrieving content - reader := a.Retrieve(key) + quitC := make(chan bool) size, err := reader.Size(quitC) if err != nil { diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go new file mode 100644 index 0000000000..2d088dfd5c --- /dev/null +++ b/swarm/api/http/server_test.go @@ -0,0 +1,108 @@ +package http + +import ( + "bytes" + "io/ioutil" + "net/http" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/logger/glog" + "github.com/ethereum/go-ethereum/swarm/api" + "github.com/ethereum/go-ethereum/swarm/storage" +) + +func TestBzzrGetPath(t *testing.T) { + + glog.SetToStderr(true) + glog.SetV(6) + + var err error + + maxproxyattempts := 3 + + testmanifest := []string{ + `{"entries":[{"path":"a/","hash":"674af7073604ebfc0282a4ab21e5ef1a3c22913866879ebc0816f8a89896b2ed","contentType":"application/bzz-manifest+json","status":0}]}`, + `{"entries":[{"path":"a","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"b/","hash":"0a87b1c3e4bf013686cdf107ec58590f2004610ee58cc2240f26939f691215f5","contentType":"application/bzz-manifest+json","status":0}]}`, + `{"entries":[{"path":"b","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0},{"path":"c","hash":"011b4d03dd8c01f1049143cf9c4c817e4b167f1d1b83e5c6f0f10d89ba1e7bce","contentType":"","status":0}]}`, + } + + testrequests := make(map[string]int) + testrequests["/"] = 0 + testrequests["/a"] = 1 + testrequests["/a/b"] = 2 + + reader := [3]*bytes.Reader{} + + key := [3]storage.Key{} + + dir, _ := ioutil.TempDir("", "bzz-storage-test") + + storeparams := &storage.StoreParams{ + dir, + 5000000, + 5000, + 0, + } + + localStore, err := storage.NewLocalStore(storage.MakeHashFunc("SHA3"), storeparams) + if err != nil { + t.Fatal(err) + } + chunker := storage.NewTreeChunker(storage.NewChunkerParams()) + dpa := &storage.DPA{ + Chunker: chunker, + ChunkStore: localStore, + } + dpa.Start() + defer dpa.Stop() + + wg := &sync.WaitGroup{} + + for i, mf := range testmanifest { + reader[i] = bytes.NewReader([]byte(mf)) + key[i], err = dpa.Store(reader[i], int64(len(mf)), wg, nil) + if err != nil { + t.Fatal(err) + } + wg.Wait() + } + + a := api.NewApi(dpa, nil) + + // iterate port numbers up if fail + StartHttpServer(a, "8504") + // how to wait for ListenAndServe to have initialized? This is pretty cruuuude + // if we fix it we don't need maxproxyattempts anymore either + time.Sleep(100 * time.Millisecond) + for i := 0; i <= maxproxyattempts; i++ { + _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") + if i == maxproxyattempts { + t.Fatalf("Failed to connect to proxy after %v attempts: %v", i, err) + } else if err != nil { + t.Logf("Proxy connect failed: %v", err) + time.Sleep(100 * time.Millisecond) + continue + } + break + } + + for k, v := range testrequests { + var body []byte + var resp *http.Response + url := "http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" + t.Logf("Sending proxy GET: %v", url) + resp, err = http.Get(url) + defer resp.Body.Close() + body, err = ioutil.ReadAll(resp.Body) + + if string(body) != testmanifest[v] { + t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(body)) + } + + t.Log(string(body)) + } + +} diff --git a/swarm/api/manifest.go b/swarm/api/manifest.go index d6dc24c486..a5076f0a7f 100644 --- a/swarm/api/manifest.go +++ b/swarm/api/manifest.go @@ -302,7 +302,8 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man if (len(path) >= epl) && (path[:epl] == entry.Path) { glog.V(logger.Detail).Infof("entry.ContentType = %v", entry.ContentType) if entry.ContentType == manifestType { - if self.loadSubTrie(entry, quitC) != nil { + err := self.loadSubTrie(entry, quitC) + if err != nil { return nil, 0 } entry, pos = entry.subtrie.findPrefixOf(path[epl:], quitC) @@ -312,9 +313,9 @@ func (self *manifestTrie) findPrefixOf(path string, quitC chan bool) (entry *man } else { pos = epl } - } else { + } /*else { entry = nil - } + }*/ return } From 6dbde1f8897319fd84b94b95e55b07be5d776d0c Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Jan 2017 23:06:36 +0200 Subject: [PATCH 11/12] swarm/api: bzzr get with path Finally a hopefully clean commit for this PR Added check for empty path to avoid SIGSEGV in path parser and resolver Added requested tests for empty path and non-existing manifest. However signature for StartHTTPServer had changed in the last time so now it's hacked as so: StartHttpServer(api.API, &Server{Addr: "127.0.0.1:8504", CorsString: ""}) --- swarm/api/api.go | 3 +++ swarm/api/http/server.go | 6 ++++- swarm/api/http/server_test.go | 47 +++++++++++++++++++++++++---------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/swarm/api/api.go b/swarm/api/api.go index 10f2597f39..f92a14653d 100644 --- a/swarm/api/api.go +++ b/swarm/api/api.go @@ -89,6 +89,9 @@ func (self *Api) Resolve(hostPort string, nameresolver bool) (storage.Key, error return contentHash[:], err } func Parse(uri string) (hostPort, path string) { + if uri == "" { + return + } parts := slashes.Split(uri, 3) var i int if len(parts) == 0 { diff --git a/swarm/api/http/server.go b/swarm/api/http/server.go index 85505776d0..afd867efc8 100644 --- a/swarm/api/http/server.go +++ b/swarm/api/http/server.go @@ -32,8 +32,8 @@ import ( "github.com/ethereum/go-ethereum/logger" "github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/api" - "github.com/rs/cors" "github.com/ethereum/go-ethereum/swarm/storage" + "github.com/rs/cors" ) const ( @@ -195,6 +195,10 @@ func handler(w http.ResponseWriter, r *http.Request, a *api.Api) { } case r.Method == "GET" || r.Method == "HEAD": path = trailingSlashes.ReplaceAllString(path, "") + if path == "" { + http.Error(w, "Empty path not allowed", http.StatusBadRequest) + return + } if raw { var reader storage.LazySectionReader parsedurl, _ := api.Parse(path) diff --git a/swarm/api/http/server_test.go b/swarm/api/http/server_test.go index 2d088dfd5c..90791c7089 100644 --- a/swarm/api/http/server_test.go +++ b/swarm/api/http/server_test.go @@ -9,15 +9,17 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/logger/glog" + //"github.com/ethereum/go-ethereum/logger/glog" "github.com/ethereum/go-ethereum/swarm/api" "github.com/ethereum/go-ethereum/swarm/storage" ) -func TestBzzrGetPath(t *testing.T) { +func init() { + //glog.SetToStderr(true) + //glog.SetV(6) +} - glog.SetToStderr(true) - glog.SetV(6) +func TestBzzrGetPath(t *testing.T) { var err error @@ -33,6 +35,10 @@ func TestBzzrGetPath(t *testing.T) { testrequests["/"] = 0 testrequests["/a"] = 1 testrequests["/a/b"] = 2 + testrequests["/x"] = 0 + testrequests[""] = 0 + + expectedfailrequests := []string{"", "/x"} reader := [3]*bytes.Reader{} @@ -73,10 +79,10 @@ func TestBzzrGetPath(t *testing.T) { a := api.NewApi(dpa, nil) // iterate port numbers up if fail - StartHttpServer(a, "8504") + StartHttpServer(a, &Server{Addr: "127.0.0.1:8504", CorsString: ""}) // how to wait for ListenAndServe to have initialized? This is pretty cruuuude // if we fix it we don't need maxproxyattempts anymore either - time.Sleep(100 * time.Millisecond) + time.Sleep(1000 * time.Millisecond) for i := 0; i <= maxproxyattempts; i++ { _, err := http.Get("http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/a") if i == maxproxyattempts { @@ -90,19 +96,34 @@ func TestBzzrGetPath(t *testing.T) { } for k, v := range testrequests { - var body []byte var resp *http.Response - url := "http://127.0.0.1:8504/bzzr:/" + common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" + var respbody []byte + + url := "http://127.0.0.1:8504/bzzr:/" + if k[:] != "" { + url += common.ToHex(key[0])[2:] + "/" + k[1:] + "?content_type=text/plain" + } t.Logf("Sending proxy GET: %v", url) resp, err = http.Get(url) defer resp.Body.Close() - body, err = ioutil.ReadAll(resp.Body) + respbody, err = ioutil.ReadAll(resp.Body) - if string(body) != testmanifest[v] { - t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(body)) + if string(respbody) != testmanifest[v] { + isexpectedfailrequest := false + + for _, r := range expectedfailrequests { + if k[:] == r { + isexpectedfailrequest = true + } + } + if isexpectedfailrequest { + t.Logf("Expected fail request failed as expected: %v", string(respbody)) + } else { + t.Fatalf("Response body does not match, expected: %v, got %v", testmanifest[v], string(respbody)) + } + } else { + t.Logf("Response body OK: %v", string(respbody)) } - - t.Log(string(body)) } } From 2f8a3dde62409678e19758fbeb3aae4079be65b5 Mon Sep 17 00:00:00 2001 From: nolash Date: Wed, 18 Jan 2017 23:44:31 +0200 Subject: [PATCH 12/12] swarm/network: cmdline network id highest precedence Already fixed in previous commits, but separate commit for this issue requested. Added comments to explain why test against 0 appears twice fixes #3444 fixes #3494 --- swarm/api/config.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/swarm/api/config.go b/swarm/api/config.go index e4c2dfb5fc..23a8555007 100644 --- a/swarm/api/config.go +++ b/swarm/api/config.go @@ -86,6 +86,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n } data, err = ioutil.ReadFile(confpath) + // if not set in function param, then set default for swarm network, will be overwritten by config file if present if networkId == 0 { self.NetworkId = network.NetworkId } @@ -117,6 +118,7 @@ func NewConfig(path string, contract common.Address, prvKey *ecdsa.PrivateKey, n return nil, fmt.Errorf("bzz key does not match the one in the config file %v != %v", keyhex, self.BzzKey) } + // if set in function param, replace id set from config file if networkId != 0 { self.NetworkId = networkId }