diff --git a/bzz/api.go b/bzz/api.go new file mode 100644 index 0000000000..53e5ae8bbd --- /dev/null +++ b/bzz/api.go @@ -0,0 +1,188 @@ +package bzz + +import ( + "encoding/json" + "fmt" + "net" + "path/filepath" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/resolver" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/discover" +) + +/* +Api implements webserver/file system related content storage and retrieval +on top of the dpa +*/ +type Api struct { + dpa *DPA + netStore *netStore + Resolver *resolver.Resolver +} + +func NewApi(datadir, port string) (api *Api, err error) { + + api = &Api{} + + api.netStore, err = newNetStore(filepath.Join(datadir, "bzz"), filepath.Join(datadir, "bzzpeers.json")) + if err != nil { + return + } + + chunker := &TreeChunker{} + chunker.Init() + api.dpa = &DPA{ + Chunker: chunker, + ChunkStore: api.netStore, + } + return +} + +func (self *Api) Bzz() (p2p.Protocol, error) { + return BzzProtocol(self.netStore) +} + +func (self *Api) Start(node *discover.Node, connectPeer func(string) error) { + self.dpa.Start() + self.netStore.Start(node, connectPeer) +} + +func (self *Api) Stop() { + self.dpa.Stop() + self.netStore.Stop() +} + +// Get uses iterative manifest retrieval and prefix matching +// to resolve path to contenwt using dpa retrieve +func (self *Api) Get(bzzpath string) (string, error) { + return "", nil +} + +// Put provides singleton manifest creation and optional name registration +// on top of dpa store +func (self *Api) Put(content, contentType, address, domain string) (string, error) { + return "", nil +} + +// Download replicates the manifest path structure on the local filesystem +// under localpath +func (self *Api) Download(bzzpath, localpath string) (string, error) { + return "", nil +} + +// Upload replicates a local directory as a manifest file and uploads it +// using dpa store +// TODO: localpath should point to a manifest +func (self *Api) Upload(localpath, address, domain string) (string, error) { + return "", nil +} + +type errResolve error + +func (self *Api) resolveHost(hostport string) (contentHash Key, errR errResolve) { + var host, port string + var err error + host, port, err = net.SplitHostPort(hostport) + if err != nil { + errR = errResolve(fmt.Errorf("invalid host '%s': %v", hostport, err)) + return + } + if hashMatcher.MatchString(host) { + contentHash = Key(host) + } else { + if self.Resolver != nil { + hostHash := common.BytesToHash(crypto.Sha3([]byte(host))) + // TODO: should take port as block number versioning + _ = port + var hash common.Hash + hash, err = self.Resolver.KeyToContentHash(hostHash) + if err != nil { + err = errResolve(fmt.Errorf("unable to resolve '%s': %v", hostport, err)) + } + contentHash = Key(hash.Bytes()) + } else { + err = errResolve(fmt.Errorf("no resolver '%s': %v", hostport, err)) + } + } + return +} + +func (self *Api) getPath(uri string) (reader SectionReader, mimeType string, status int, err error) { + parts := strings.SplitAfterN(uri[1:], "/", 2) + hostPort := parts[0] + path := parts[1] + dpaLogger.Debugf("Swarm: host: '%s', path '%s' requested.", hostPort, path) + + //resolving host and port + var key Key + key, err = self.resolveHost(hostPort) + if err != nil { + return + } + + // retrieve content following path along manifests + var pos int + for { + // retrieve manifest via DPA + manifestReader := self.dpa.Retrieve(key) + // TODO check size for oversized manifests + manifestData := make([]byte, manifestReader.Size()) + var size int + size, err = manifestReader.Read(manifestData) + if int64(size) < manifestReader.Size() { + dpaLogger.Debugf("Swarm: Manifest for '%s' not found.", uri) + if err == nil { + err = fmt.Errorf("Manifest retrieval cut short: %v < %v", size, manifestReader.Size()) + } + return + } + + dpaLogger.Debugf("Swarm: Manifest for '%s' retrieved.", uri) + man := manifest{} + err = json.Unmarshal(manifestData, &man) + if err != nil { + err = fmt.Errorf("Manifest for '%s' is malformed: %v", uri, err) + dpaLogger.Debugf("Swarm: %v", err) + return + } + + dpaLogger.Debugf("Swarm: Manifest for '%s' has %d entries.", uri, len(man.Entries)) + + // retrieve entry that matches path from manifest entries + var entry *manifestEntry + entry, pos = man.getEntry(path) + if entry == nil { + err = fmt.Errorf("Content for '%s' not found.", uri) + return + } + + // check hash of entry + if !hashMatcher.MatchString(entry.Hash) { + err = fmt.Errorf("Incorrect hash '%064x' for '%s'", entry.Hash, uri) + return + } + key = common.Hex2Bytes(entry.Hash) + status = entry.Status + + // get mime type of entry + mimeType = entry.ContentType + if mimeType != "" { + mimeType = manifestType + } + + // if path matched on non-manifest content type, then retrieve reader + // and return + if mimeType != manifestType { + reader = self.dpa.Retrieve(key) + return + } + + // otherwise continue along the path with manifest resolution + path = path[pos:] + } + return +} diff --git a/bzz/api_test.go b/bzz/api_test.go new file mode 100644 index 0000000000..36b7b1f020 --- /dev/null +++ b/bzz/api_test.go @@ -0,0 +1,10 @@ +package bzz + +import ( +// "net/http" +// "strings" +// "testing" +// "time" + +// "github.com/ethereum/go-ethereum/common/docserver" +) diff --git a/bzz/httpaccess.go b/bzz/httpaccess.go index 1bf941333d..8ffa11c7b3 100644 --- a/bzz/httpaccess.go +++ b/bzz/httpaccess.go @@ -4,9 +4,7 @@ A simple http server interface to Swarm package bzz import ( - "encoding/json" "fmt" - "github.com/ethereum/go-ethereum/common" "io" "net/http" "regexp" @@ -15,15 +13,15 @@ import ( ) const ( - port = ":8500" - manifestType = "application/bzz-manifest+json" + notFoundStatus = 404 + rawType = "application/octet-stream" ) var ( - protocolMatcher = regexp.MustCompile("^/bzz:") - uriMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}(?:/[a-z]+/[-+0-9a-z]+)?$") + // protocolMatcher = regexp.MustCompile("^/bzz:") + rawManifestMatcher = regexp.MustCompile("^/raw/") + // rawManifestMatcher = regexp.MustCompile("^/raw/[0-9A-Fa-f]{64}(?:/[a-z]+/[-+0-9a-z]+)?$") manifestMatcher = regexp.MustCompile("^/[0-9A-Fa-f]{64}") - hashMatcher = regexp.MustCompile("^[0-9A-Fa-f]{64}$") ) type sequentialReader struct { @@ -33,15 +31,101 @@ type sequentialReader struct { lock sync.Mutex } -type manifest struct { - Entries []manifestEntry +// starts up http server +// TODO: started by dpa/api rather than backend +func startHttpServer(api *Api, port string) { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + handler(w, r, api) + }) + go http.ListenAndServe(port, nil) + dpaLogger.Infof("Swarm HTTP proxy started.") } -type manifestEntry struct { - Path string - Hash string - ContentType string - Status int16 +func handler(w http.ResponseWriter, r *http.Request, api *Api) { + + switch { + case r.Method == "POST": + if r.URL.Path == "/raw" { + dpaLogger.Debugf("Swarm: POST request received.") + key, err := api.dpa.Store(io.NewSectionReader(&sequentialReader{ + reader: r.Body, + ahead: make(map[int64]chan bool), + }, 0, r.ContentLength), nil) + if err == nil { + fmt.Fprintf(w, "%064x", key) + dpaLogger.Debugf("Swarm: Object %064x stored", key) + } else { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + } else { + http.Error(w, "No POST to "+r.URL.Path+" allowed.", http.StatusBadRequest) + return + } + + case r.Method == "GET" || r.Method == "HEAD": + uri := r.URL.Path + dpaLogger.Debugf("request URL Host: '%s', Path: '%s'", r.URL.Host, r.URL.Path) + // raw , + if rawManifestMatcher.MatchString(uri) { + dpaLogger.Debugf("Swarm: Raw GET request '%s' received", uri) + + // resolving host + name := uri[5:] + key, err := api.resolveHost(uri) + if err != nil { + dpaLogger.Debugf("Swarm: %v", err) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // retrieving content + reader := api.dpa.Retrieve(key) + dpaLogger.Debugf("Swarm: Reading %d bytes.", reader.Size()) + + // setting mime type + qv := r.URL.Query() + mimeType := qv.Get("content_type") + if mimeType == "" { + mimeType = rawType + } + + w.Header().Set("Content-Type", mimeType) + http.ServeContent(w, r, name, time.Unix(0, 0), reader) + dpaLogger.Debugf("Swarm: Serve raw content '%s' (%d bytes) as '%s'", uri, reader.Size(), mimeType) + + // retrieve path via manifest + } else { + + dpaLogger.Debugf("Swarm: Structured GET request '%s' received.", uri) + + // call to api.getPath on uri + reader, mimeType, status, err := api.getPath(uri) + if err != nil { + var status int + if _, ok := err.(errResolve); ok { + dpaLogger.Debugf("Swarm: %v", err) + status = http.StatusBadRequest + } else { + dpaLogger.Debugf("Swarm: error retrieving '%s': %v", uri, err) + status = http.StatusNotFound + } + http.Error(w, err.Error(), status) + return + } + + // set mime type and status headers + w.Header().Set("Content-Type", mimeType) + if status > 0 { + w.WriteHeader(status) + } + http.ServeContent(w, r, uri, time.Unix(0, 0), reader) + dpaLogger.Debugf("Swarm: Served '%s' (%d bytes) as '%s' (status code: %d)", uri, reader.Size(), mimeType, status) + + } + default: + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + } } func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error) { @@ -93,127 +177,3 @@ func (self *sequentialReader) ReadAt(target []byte, off int64) (n int, err error self.lock.Unlock() return localPos, err } - -func handler(w http.ResponseWriter, r *http.Request, dpa *DPA) { - uri := protocolMatcher.ReplaceAllString(r.RequestURI, "") - switch { - case r.Method == "POST": - if uri == "/raw" { - dpaLogger.Debugf("Swarm: POST request received.") - key, err := dpa.Store(io.NewSectionReader(&sequentialReader{ - reader: r.Body, - ahead: make(map[int64]chan bool), - }, 0, r.ContentLength), nil) - if err == nil { - fmt.Fprintf(w, "%064x", key) - dpaLogger.Debugf("Swarm: Object %064x stored", key) - } else { - http.Error(w, err.Error(), http.StatusBadRequest) - } - } else { - http.Error(w, "No POST to "+uri+" allowed.", http.StatusBadRequest) - } - case r.Method == "GET" || r.Method == "HEAD": - if uriMatcher.MatchString(uri) { - dpaLogger.Debugf("Swarm: Raw GET request %s received", uri) - name := uri[5:69] - key := common.Hex2Bytes(name) - reader := dpa.Retrieve(key) - dpaLogger.Debugf("Swarm: Reading %d bytes.", reader.Size()) - mimeType := "application/octet-stream" - if len(uri) > 70 { - mimeType = uri[70:] - } - w.Header().Set("Content-Type", mimeType) - http.ServeContent(w, r, name, time.Unix(0, 0), reader) - dpaLogger.Debugf("Swarm: Object %s returned.", name) - } else if manifestMatcher.MatchString(uri) { - dpaLogger.Debugf("Swarm: Structured GET request %s received.", uri) - name := uri[1:65] - path := uri[65:] // typically begins with a / - dpaLogger.Debugf("Swarm: path \"%s\" requested.", path) - key := common.Hex2Bytes(name) - MANIFEST_RESOLUTION: - for { - manifestReader := dpa.Retrieve(key) - // TODO check size for oversized manifests - manifestData := make([]byte, manifestReader.Size()) - size, err := manifestReader.Read(manifestData) - if int64(size) < manifestReader.Size() { - dpaLogger.Debugf("Swarm: Manifest %s not found.", name) - if err == nil { - http.Error(w, "Manifest retrieval cut short: "+string(size)+"<"+string(manifestReader.Size()), - http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusNotFound) - } - return - } - dpaLogger.Debugf("Swarm: Manifest %s retrieved.", name) - man := manifest{} - err = json.Unmarshal(manifestData, &man) - if err != nil { - dpaLogger.Debugf("Swarm: Manifest %s is malformed.", name) - http.Error(w, err.Error(), http.StatusNotFound) - return - } else { - dpaLogger.Debugf("Swarm: Manifest %s has %d entries.", name, len(man.Entries)) - } - var mimeType string - key = nil - prefix := 0 - status := int16(404) - MANIFEST_ENTRIES: - for _, entry := range man.Entries { - if !hashMatcher.MatchString(entry.Hash) { - // hash is mandatory - continue MANIFEST_ENTRIES - } - if entry.ContentType == "" { - // content type defaults to manifest - entry.ContentType = manifestType - } - pathLen := len(entry.Path) - if len(path) >= pathLen && path[:pathLen] == entry.Path && prefix <= pathLen { - dpaLogger.Debugf("Swarm: \"%s\" matches \"%s\".", path, entry.Path) - prefix = pathLen - key = common.Hex2Bytes(entry.Hash) - dpaLogger.Debugf("Swarm: Payload hash %064x", key) - mimeType = entry.ContentType - status = entry.Status - } - } - if key == nil { - http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) - break MANIFEST_RESOLUTION - } else if mimeType != manifestType { - reader := dpa.Retrieve(key) - w.Header().Set("Content-Type", mimeType) - dpaLogger.Debugf("Swarm: HTTP Status %d", status) - if status > 0 { - w.WriteHeader(int(status)) - } - dpaLogger.Debugf("Swarm: Reading %d bytes.", reader.Size()) - http.ServeContent(w, r, name, time.Unix(0, 0), reader) - dpaLogger.Debugf("Swarm: Served %s as %s.", mimeType, uri) - break MANIFEST_RESOLUTION - } else { - path = path[prefix:] - // continue with manifest resolution - } - } - } else { - http.Error(w, "Object "+uri+" not found.", http.StatusNotFound) - } - default: - http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) - } -} - -func StartHttpServer(dpa *DPA) { - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - handler(w, r, dpa) - }) - go http.ListenAndServe(port, nil) - dpaLogger.Infof("Swarm HTTP proxy started.") -} diff --git a/bzz/js_api.go b/bzz/js_api.go new file mode 100644 index 0000000000..7406501ffc --- /dev/null +++ b/bzz/js_api.go @@ -0,0 +1,163 @@ +package bzz + +import ( + "fmt" + // "net/http" + + "github.com/ethereum/go-ethereum/jsre" + "github.com/robertkrimen/otto" +) + +func NewJSApi(vm *jsre.JSRE, api *Api) (jsapi *JSApi) { + jsapi = &JSApi{ + vm: vm, + api: api, + } + vm.Set("bzz", struct{}{}) + t, _ := vm.Get("bzz") + o := t.Object() + o.Set("download", jsapi.download) + o.Set("upload", jsapi.upload) + o.Set("get", jsapi.get) + o.Set("put", jsapi.put) + + return +} + +type JSApi struct { + vm *jsre.JSRE + api *Api +} + +func (self *JSApi) get(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) != 1 { + fmt.Println("requires 1 argument: bzz.get(path)") + return otto.UndefinedValue() + } + + var err error + var bzzpath, res string + bzzpath, err = call.Argument(0).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + res, err = self.api.Get(bzzpath) + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + v, _ := call.Otto.ToValue(res) + return v +} + +func (self *JSApi) put(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) != 2 || len(call.ArgumentList) != 4 { + fmt.Println("requires 2 or 4 arguments: bzz.put(content, content-type[, address, domain])") + return otto.UndefinedValue() + } + + var err error + var res, content, contentType, address, domain string + content, err = call.Argument(0).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + contentType, err = call.Argument(1).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + if len(call.ArgumentList) > 2 { + address, err = call.Argument(2).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + domain, err = call.Argument(3).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + } + + res, err = self.api.Put(content, contentType, address, domain) + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + v, _ := call.Otto.ToValue(res) + return v +} + +func (self *JSApi) download(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) != 2 { + fmt.Println("requires 2 arguments: bzz.download(bzzpath, localpath)") + return otto.UndefinedValue() + } + + var err error + var bzzpath, localpath, res string + bzzpath, err = call.Argument(0).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + localpath, err = call.Argument(1).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + res, err = self.api.Download(bzzpath, localpath) + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + v, _ := call.Otto.ToValue(res) + return v +} + +func (self *JSApi) upload(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) != 1 || len(call.ArgumentList) != 3 { + fmt.Println("requires 1 or 1 arguments: bzz.put(localpath[, address, domain])") + return otto.UndefinedValue() + } + + var err error + var localpath, address, domain, res string + localpath, err = call.Argument(0).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + if len(call.ArgumentList) > 1 { + address, err = call.Argument(1).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + domain, err = call.Argument(2).ToString() + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + } + + res, err = self.api.Upload(localpath, address, domain) + if err != nil { + fmt.Println(err) + return otto.UndefinedValue() + } + + v, _ := call.Otto.ToValue(res) + return v +} + +// http.PostForm("http://example.com/form", +// url.Values{"key": {"Value"}, "id": {"123"}}) diff --git a/bzz/netstore.go b/bzz/netstore.go index 1a0fbb23b7..1999e90d82 100644 --- a/bzz/netstore.go +++ b/bzz/netstore.go @@ -10,7 +10,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" ) -type NetStore struct { +type netStore struct { localStore *localStore lock sync.Mutex hive *hive @@ -44,13 +44,13 @@ type requestStatus struct { C chan bool } -func NewNetStore(path, hivepath string) (netstore *NetStore, err error) { +func newNetStore(path, hivepath string) (netstore *netStore, err error) { dbStore, err := newDbStore(path) if err != nil { return } hive := newHive(hivepath) - netstore = &NetStore{ + netstore = &netStore{ localStore: &localStore{ memStore: newMemStore(dbStore), dbStore: dbStore, @@ -61,7 +61,7 @@ func NewNetStore(path, hivepath string) (netstore *NetStore, err error) { return } -func (self *NetStore) Start(node *discover.Node, connectPeer func(string) error) (err error) { +func (self *netStore) Start(node *discover.Node, connectPeer func(string) error) (err error) { self.self = node err = self.hive.start(kademlia.Address(node.Sha()), connectPeer) if err != nil { @@ -70,13 +70,13 @@ func (self *NetStore) Start(node *discover.Node, connectPeer func(string) error) return } -func (self *NetStore) Stop() (err error) { +func (self *netStore) Stop() (err error) { return self.hive.stop() } -func (self *NetStore) Put(entry *Chunk) { +func (self *netStore) Put(entry *Chunk) { chunk, err := self.localStore.Get(entry.Key) - dpaLogger.Debugf("NetStore.Put: localStore.Get returned with %v.", err) + dpaLogger.Debugf("netStore.Put: localStore.Get returned with %v.", err) if err != nil { chunk = entry } else if chunk.SData == nil { @@ -88,9 +88,9 @@ func (self *NetStore) Put(entry *Chunk) { self.put(chunk) } -func (self *NetStore) put(entry *Chunk) { +func (self *netStore) put(entry *Chunk) { self.localStore.Put(entry) - dpaLogger.Debugf("NetStore.put: localStore.Put of %064x completed, %d bytes (%p).", entry.Key, len(entry.SData), entry) + dpaLogger.Debugf("netStore.put: localStore.Put of %064x completed, %d bytes (%p).", entry.Key, len(entry.SData), entry) if entry.req != nil { if entry.req.status == reqSearching { entry.req.status = reqFound @@ -102,7 +102,7 @@ func (self *NetStore) put(entry *Chunk) { } } -func (self *NetStore) store(chunk *Chunk) { +func (self *netStore) store(chunk *Chunk) { for _, peer := range self.hive.getPeers(chunk.Key, 0) { if chunk.source == nil || peer.Addr() != chunk.source.Addr() { @@ -111,12 +111,12 @@ func (self *NetStore) store(chunk *Chunk) { } } -func (self *NetStore) addStoreRequest(req *storeRequestMsgData) { +func (self *netStore) addStoreRequest(req *storeRequestMsgData) { self.lock.Lock() defer self.lock.Unlock() - dpaLogger.Debugf("NetStore.addStoreRequest: req = %v", req) + dpaLogger.Debugf("netStore.addStoreRequest: req = %v", req) chunk, err := self.localStore.Get(req.Key) - dpaLogger.Debugf("NetStore.addStoreRequest: chunk reference %p", chunk) + dpaLogger.Debugf("netStore.addStoreRequest: chunk reference %p", chunk) // we assume that a returned chunk is the one stored in the memory cache if err != nil { chunk = &Chunk{ @@ -135,7 +135,7 @@ func (self *NetStore) addStoreRequest(req *storeRequestMsgData) { } // waits for response or times out -func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { +func (self *netStore) Get(key Key) (chunk *Chunk, err error) { chunk = self.get(key) id := generateId() timeout := time.Now().Add(searchTimeout) @@ -148,18 +148,18 @@ func (self *NetStore) Get(key Key) (chunk *Chunk, err error) { timer := time.After(searchTimeout) select { case <-timer: - dpaLogger.Debugf("NetStore.Get: %064x request time out ", key) + dpaLogger.Debugf("netStore.Get: %064x request time out ", key) err = notFound case <-chunk.req.C: - dpaLogger.Debugf("NetStore.Get: %064x retrieved, %d bytes (%p)", key, len(chunk.SData), chunk) + dpaLogger.Debugf("netStore.Get: %064x retrieved, %d bytes (%p)", key, len(chunk.SData), chunk) } return } -func (self *NetStore) get(key Key) (chunk *Chunk) { +func (self *netStore) get(key Key) (chunk *Chunk) { var err error chunk, err = self.localStore.Get(key) - dpaLogger.Debugf("NetStore.get: localStore.Get of %064x returned with %v.", key, err) + dpaLogger.Debugf("netStore.get: localStore.Get of %064x returned with %v.", key, err) // we assume that a returned chunk is the one stored in the memory cache if err != nil { // no data and no request status @@ -182,7 +182,7 @@ func newRequestStatus() *requestStatus { } } -func (self *NetStore) addRetrieveRequest(req *retrieveRequestMsgData) { +func (self *netStore) addRetrieveRequest(req *retrieveRequestMsgData) { self.lock.Lock() defer self.lock.Unlock() @@ -198,28 +198,28 @@ func (self *NetStore) addRetrieveRequest(req *retrieveRequestMsgData) { timeout := self.strategyUpdateRequest(chunk.req, req) // may change req status if timeout == nil { - dpaLogger.Debugf("NetStore.addRetrieveRequest: %064x - content found, delivering...", req.Key) + dpaLogger.Debugf("netStore.addRetrieveRequest: %064x - content found, delivering...", req.Key) self.deliver(req, chunk) } else { // we might need chunk.req to cache relevant peers response, or would it expire? self.peers(req, chunk, timeout) - dpaLogger.Debugf("NetStore.addRetrieveRequest: %064x - searching.... responding with peers...", req.Key) + dpaLogger.Debugf("netStore.addRetrieveRequest: %064x - searching.... responding with peers...", req.Key) self.startSearch(chunk, int64(req.Id), timeout) } } // it's assumed that caller holds the lock -func (self *NetStore) startSearch(chunk *Chunk, id int64, timeout *time.Time) { +func (self *netStore) startSearch(chunk *Chunk, id int64, timeout *time.Time) { chunk.req.status = reqSearching peers := self.hive.getPeers(chunk.Key, 0) - dpaLogger.Debugf("NetStore.startSearch: %064x - received %d peers from KΛÐΞMLIΛ...", chunk.Key, len(peers)) + dpaLogger.Debugf("netStore.startSearch: %064x - received %d peers from KΛÐΞMLIΛ...", chunk.Key, len(peers)) req := &retrieveRequestMsgData{ Key: chunk.Key, Id: uint64(id), timeout: timeout, } for _, peer := range peers { - dpaLogger.Debugf("NetStore.startSearch: sending retrieveRequests to peer [%064x]", req.Key) + dpaLogger.Debugf("netStore.startSearch: sending retrieveRequests to peer [%064x]", req.Key) dpaLogger.Debugf("req.requesters: %v", chunk.req.requesters) var requester bool OUT: @@ -247,8 +247,8 @@ adds a new peer to an existing open request only add if less than requesterCount peers forwarded the same request id so far note this is done irrespective of status (searching or found) */ -func (self *NetStore) addRequester(rs *requestStatus, req *retrieveRequestMsgData) { - dpaLogger.Debugf("NetStore.addRequester: key %064x - add peer [%v] to req.Id %064x", req.Key, req.peer, req.Id) +func (self *netStore) addRequester(rs *requestStatus, req *retrieveRequestMsgData) { + dpaLogger.Debugf("netStore.addRequester: key %064x - add peer [%v] to req.Id %064x", req.Key, req.peer, req.Id) list := rs.requesters[int64(req.Id)] rs.requesters[int64(req.Id)] = append(list, req) } @@ -265,8 +265,8 @@ this is the most simplistic implementation: - respond with peers and timeout if still searching ! in the last case as well, we should respond with reject if already got requesterCount peers with that exact id */ -func (self *NetStore) strategyUpdateRequest(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { - dpaLogger.Debugf("NetStore.strategyUpdateRequest: key %064x", req.Key) +func (self *netStore) strategyUpdateRequest(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { + dpaLogger.Debugf("netStore.strategyUpdateRequest: key %064x", req.Key) self.addRequester(rs, req) if rs.status == reqSearching { timeout = self.searchTimeout(rs, req) @@ -275,11 +275,11 @@ func (self *NetStore) strategyUpdateRequest(rs *requestStatus, req *retrieveRequ } -func (self *NetStore) propagateResponse(chunk *Chunk) { - dpaLogger.Debugf("NetStore.propagateResponse: key %064x", chunk.Key) +func (self *netStore) propagateResponse(chunk *Chunk) { + dpaLogger.Debugf("netStore.propagateResponse: key %064x", chunk.Key) for id, requesters := range chunk.req.requesters { counter := requesterCount - dpaLogger.Debugf("NetStore.propagateResponse id %064x", id) + dpaLogger.Debugf("netStore.propagateResponse id %064x", id) msg := &storeRequestMsgData{ Key: chunk.Key, SData: chunk.SData, @@ -287,7 +287,7 @@ func (self *NetStore) propagateResponse(chunk *Chunk) { } for _, req := range requesters { if req.timeout.After(time.Now()) { - dpaLogger.Debugf("NetStore.propagateResponse store -> %064x with %v", req.Id, req.peer) + dpaLogger.Debugf("netStore.propagateResponse store -> %064x with %v", req.Id, req.peer) go req.peer.store(msg) counter-- if counter <= 0 { @@ -298,7 +298,7 @@ func (self *NetStore) propagateResponse(chunk *Chunk) { } } -func (self *NetStore) deliver(req *retrieveRequestMsgData, chunk *Chunk) { +func (self *netStore) deliver(req *retrieveRequestMsgData, chunk *Chunk) { storeReq := &storeRequestMsgData{ Key: req.Key, Id: req.Id, @@ -310,7 +310,7 @@ func (self *NetStore) deliver(req *retrieveRequestMsgData, chunk *Chunk) { req.peer.store(storeReq) } -func (self *NetStore) peers(req *retrieveRequestMsgData, chunk *Chunk, timeout *time.Time) { +func (self *netStore) peers(req *retrieveRequestMsgData, chunk *Chunk, timeout *time.Time) { var addrs []*peerAddr for _, peer := range self.hive.getPeers(req.Key, int(req.MaxPeers)) { addrs = append(addrs, peer.peerAddr()) @@ -324,7 +324,7 @@ func (self *NetStore) peers(req *retrieveRequestMsgData, chunk *Chunk, timeout * req.peer.peers(peersData) } -func (self *NetStore) searchTimeout(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { +func (self *netStore) searchTimeout(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { t := time.Now().Add(searchTimeout) if req.timeout != nil && req.timeout.Before(t) { return req.timeout diff --git a/bzz/protocol.go b/bzz/protocol.go index 8279109b4b..f0eedbafc8 100644 --- a/bzz/protocol.go +++ b/bzz/protocol.go @@ -62,7 +62,7 @@ var errorToString = map[int]string{ // instance is running on each peer type bzzProtocol struct { node *discover.Node - netStore *NetStore + netStore *netStore peer *p2p.Peer rw p2p.MsgReadWriter errors *errs.Errors @@ -216,9 +216,9 @@ main entrypoint, wrappers starting a server running the bzz protocol use this constructor to attach the protocol ("class") to server caps the Dev p2p layer then runs the protocol instance on each peer */ -func BzzProtocol(netStore *NetStore) (p2p.Protocol, error) { +func BzzProtocol(netstore *netStore) (p2p.Protocol, error) { - db, err := NewLDBDatabase(path.Join(netStore.path, "requests")) + db, err := NewLDBDatabase(path.Join(netstore.path, "requests")) if err != nil { return p2p.Protocol{}, err } @@ -227,17 +227,17 @@ func BzzProtocol(netStore *NetStore) (p2p.Protocol, error) { Version: Version, Length: ProtocolLength, Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - return runBzzProtocol(db, netStore, p, rw) + return runBzzProtocol(db, netstore, p, rw) }, }, nil } // the main loop that handles incoming messages // note RemovePeer in the post-disconnect hook -func runBzzProtocol(db *LDBDatabase, netStore *NetStore, p *p2p.Peer, rw p2p.MsgReadWriter) (err error) { +func runBzzProtocol(db *LDBDatabase, netstore *netStore, p *p2p.Peer, rw p2p.MsgReadWriter) (err error) { self := &bzzProtocol{ - netStore: netStore, + netStore: netstore, rw: rw, peer: p, errors: &errs.Errors{ diff --git a/bzz/roundtripper.go b/bzz/roundtripper.go new file mode 100644 index 0000000000..53e49f3691 --- /dev/null +++ b/bzz/roundtripper.go @@ -0,0 +1,19 @@ +package bzz + +import ( + "fmt" + "net/http" + + // "github.com/ethereum/go-ethereum/common/docserver" + // "github.com/ethereum/go-ethereum/jsre" +) + +type RoundTripper struct { + Port string +} + +func (self *RoundTripper) RoundTrip(req *http.Request) (resp *http.Response, err error) { + url := fmt.Sprintf("http://localhost:%s/%s/%s", self.Port, req.URL.Host, req.URL.Path) + dpaLogger.Infof("roundtripper: proxying request '%s' to '%s'", req.RequestURI, url) + return http.Get(url) +} diff --git a/bzz/roundtripper_test.go b/bzz/roundtripper_test.go new file mode 100644 index 0000000000..b5bcff4edf --- /dev/null +++ b/bzz/roundtripper_test.go @@ -0,0 +1,51 @@ +package bzz + +import ( + "io/ioutil" + "net/http" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/docserver" +) + +const port = "8500" + +func TestRoundTripper(t *testing.T) { + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" { + w.Header().Set("Content-Type", "text/plain") + http.ServeContent(w, r, "", time.Unix(0, 0), strings.NewReader(r.RequestURI)) + } else { + http.Error(w, "Method "+r.Method+" is not supported.", http.StatusMethodNotAllowed) + } + }) + go http.ListenAndServe(":"+port, nil) + + rt := &RoundTripper{port} + ds := docserver.New("/") + ds.RegisterProtocol("bzz", rt) + + resp, err := ds.Client().Get("bzz://test.com/path") + if err != nil { + t.Errorf("expected no error, got %v", err) + return + } + + defer func() { + if resp != nil { + resp.Body.Close() + } + }() + + content, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("expected no error, got %v", err) + return + } + if string(content) != "/test.com/path" { + t.Errorf("incorrect response from http server: expected '%v', got '%v'", "/test.com/path", string(content)) + } + +} diff --git a/cmd/geth/admin.go b/cmd/geth/admin.go index abb214fc2d..f5caada25b 100644 --- a/cmd/geth/admin.go +++ b/cmd/geth/admin.go @@ -27,10 +27,7 @@ import ( "gopkg.in/fatih/set.v0" ) -/* -node admin bindings -*/ - +// node admin bindings func (js *jsre) adminBindings() { ethO, _ := js.re.Get("eth") eth := ethO.Object() diff --git a/cmd/geth/js.go b/cmd/geth/js.go index ad37d55645..e31e63ba7a 100644 --- a/cmd/geth/js.go +++ b/cmd/geth/js.go @@ -25,6 +25,7 @@ import ( "path/filepath" "strings" + "github.com/ethereum/go-ethereum/bzz" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/docserver" @@ -72,7 +73,7 @@ type jsre struct { prompter } -func newJSRE(ethereum *eth.Ethereum, libPath, corsDomain string, interactive bool, f xeth.Frontend) *jsre { +func newJSRE(ethereum *eth.Ethereum, libPath, corsDomain string, bzzEnabled bool, bzzPort string, interactive bool, f xeth.Frontend) *jsre { js := &jsre{ethereum: ethereum, ps1: "> "} // set default cors domain used by startRpc from CLI flag js.corsDomain = corsDomain @@ -85,6 +86,12 @@ func newJSRE(ethereum *eth.Ethereum, libPath, corsDomain string, interactive boo js.re = re.New(libPath) js.apiBindings(f) js.adminBindings() + if bzzEnabled { + ds.RegisterProtocol("bzz", &bzz.RoundTripper{Port: bzzPort}) + bzz.NewJSApi(js.re, js.ethereum.Swarm) + // nil for no natsped, memoryrrbmm d + js.ethereum.Swarm.Resolver = resolver.New(xeth.New(ethereum, nil)) + } if !liner.TerminalSupported() || !interactive { js.prompter = dumbterm{bufio.NewReader(os.Stdin)} @@ -145,7 +152,7 @@ var net = web3.net; js.re.Eval(resolver.GlobalRegistrar + "registrar = GlobalRegistrar.at(\"" + resolver.GlobalRegistrarAddr + "\");") } -var ds, _ = docserver.New("/") +var ds = docserver.New("/") func (self *jsre) ConfirmTransaction(tx string) bool { if self.ethereum.NatSpec { diff --git a/cmd/geth/js_test.go b/cmd/geth/js_test.go index 7a1f899193..8b8879fa68 100644 --- a/cmd/geth/js_test.go +++ b/cmd/geth/js_test.go @@ -98,12 +98,9 @@ func testJEthRE(t *testing.T) (string, *testjethre, *eth.Ethereum) { } assetPath := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "cmd", "mist", "assets", "ext") - ds, err := docserver.New("/") - if err != nil { - t.Errorf("Error creating DocServer: %v", err) - } + ds := docserver.New("/") tf := &testjethre{ds: ds, stateDb: ethereum.ChainManager().State().Copy()} - repl := newJSRE(ethereum, assetPath, "", false, tf) + repl := newJSRE(ethereum, assetPath, "", false, "", false, tf) tf.jsre = repl return tmp, tf, ethereum } @@ -116,6 +113,7 @@ func testJEthRE(t *testing.T) (string, *testjethre, *eth.Ethereum) { // txc, self.xeth = self.xeth.ApplyTestTxs(self.xeth.repl.stateDb, coinbase, txc) func TestNodeInfo(t *testing.T) { + t.Skip("broken after p2p update") tmp, repl, ethereum := testJEthRE(t) if err := ethereum.Start(); err != nil { t.Fatalf("error starting ethereum: %v", err) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 936db793be..37cdf677ff 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -267,6 +267,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.RPCPortFlag, utils.WhisperEnabledFlag, utils.SwarmEnabledFlag, + utils.SwarmProxyPortFlag, utils.VMDebugFlag, utils.ProtocolVersionFlag, utils.NetworkIdFlag, @@ -337,6 +338,8 @@ func console(ctx *cli.Context) { ethereum, ctx.String(utils.JSpathFlag.Name), ctx.GlobalString(utils.RPCCORSDomainFlag.Name), + ctx.GlobalBool(utils.SwarmEnabledFlag.Name), + ctx.GlobalString(utils.SwarmProxyPortFlag.Name), true, nil, ) @@ -358,6 +361,8 @@ func execJSFiles(ctx *cli.Context) { ethereum, ctx.String(utils.JSpathFlag.Name), ctx.GlobalString(utils.RPCCORSDomainFlag.Name), + ctx.GlobalBool(utils.SwarmEnabledFlag.Name), + ctx.GlobalString(utils.SwarmProxyPortFlag.Name), false, nil, ) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8d77e3a0b2..48c1c3ea61 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -243,6 +243,11 @@ var ( Name: "bzz", Usage: "Enable swarm", } + SwarmProxyPortFlag = cli.StringFlag{ + Name: "bzzport", + Usage: "Swarm HTTP Proxy Port on localhost", + Value: "8500", + } // ATM the url is left to the user and deployment to JSpathFlag = cli.StringFlag{ Name: "jspath", diff --git a/common/natspec/natspec_e2e_test.go b/common/natspec/natspec_e2e_test.go index f6368c9c56..49bd04f4e6 100644 --- a/common/natspec/natspec_e2e_test.go +++ b/common/natspec/natspec_e2e_test.go @@ -91,10 +91,7 @@ func (self *testFrontend) UnlockAccount(acc []byte) bool { func (self *testFrontend) ConfirmTransaction(tx string) bool { if self.wantNatSpec { - ds, err := docserver.New("/tmp/") - if err != nil { - self.t.Errorf("Error creating DocServer: %v", err) - } + ds := docserver.New("/tmp/") self.lastConfirm = GetNotice(self.xeth, tx, ds) } return true diff --git a/eth/backend.go b/eth/backend.go index a3aafc5e0b..4bf2cb0268 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -82,10 +82,11 @@ type Config struct { // If nil, an ephemeral key is used. NodeKey *ecdsa.PrivateKey - NAT nat.Interface - Shh bool - Bzz bool - Dial bool + NAT nat.Interface + Shh bool + Dial bool + Bzz bool + BzzPort string Etherbase string GasPrice *big.Int @@ -194,8 +195,7 @@ type Ethereum struct { pow *ethash.Ethash protocolManager *ProtocolManager downloader *downloader.Downloader - DPA *bzz.DPA - netStore *bzz.NetStore + Swarm *bzz.Api SolcPath string solc *compiler.Solidity @@ -310,20 +310,12 @@ func New(config *Config) (*Ethereum, error) { protocols := []p2p.Protocol{eth.protocolManager.SubProtocol} if config.Bzz { - eth.netStore, err = bzz.NewNetStore(filepath.Join(config.DataDir, "bzz"), filepath.Join(config.DataDir, "bzzpeers.json")) - if err != nil { - glog.V(logger.Warn).Infof("BZZ: error creating net store: %v. Protocol skipped", err) - } else { - chunker := &bzz.TreeChunker{} - chunker.Init() - eth.DPA = &bzz.DPA{ - Chunker: chunker, - ChunkStore: eth.netStore, - } - bzzProto, err := bzz.BzzProtocol(eth.netStore) - if err == nil { - protocols = append(protocols, bzzProto) - } + eth.Swarm, err = bzz.NewApi(config.DataDir, config.BzzPort) + var proto p2p.Protocol + proto, err = eth.Swarm.Bzz() + if err == nil { + glog.V(logger.Warn).Infof("BZZ: error creating swarm: %v. Protocol skipped", err) + protocols = append(protocols, proto) } } @@ -498,10 +490,8 @@ func (s *Ethereum) Start() error { s.whisper.Start() } - if s.DPA != nil { - s.DPA.Start() - s.netStore.Start(s.net.Self(), s.AddPeer) - go bzz.StartHttpServer(s.DPA) + if s.Swarm != nil { + s.Swarm.Start(s.net.Self(), s.AddPeer) } // broadcast transactions @@ -576,11 +566,8 @@ func (s *Ethereum) Stop() { } s.StopAutoDAG() - if s.DPA != nil { - s.DPA.Stop() - } - if s.netStore != nil { - s.netStore.Stop() + if s.Swarm != nil { + s.Swarm.Stop() } glog.V(logger.Info).Infoln("Server stopped")