From 49d43997114db2d1a31a6d7cb15544f8c54e2cf7 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 29 May 2015 11:23:38 +0100 Subject: [PATCH] improved documentation and unexport netStore start/stop --- bzz/api.go | 31 +++++++++++++++------- bzz/dpa.go | 3 ++- bzz/netstore.go | 43 ++++++++++++++++++------------- cmd/geth/js.go | 3 ++- common/resolver/resolver.go | 51 ++++++++++++++++++++++++++++--------- 5 files changed, 91 insertions(+), 40 deletions(-) diff --git a/bzz/api.go b/bzz/api.go index 8322252519..49931c5106 100644 --- a/bzz/api.go +++ b/bzz/api.go @@ -27,6 +27,7 @@ var ( /* Api implements webserver/file system related content storage and retrieval on top of the dpa +it is the public interface of the dpa which is included in the ethereum stack */ type Api struct { dpa *DPA @@ -35,6 +36,12 @@ type Api struct { Resolver *resolver.Resolver } +/* +the api constructor initialises +- the netstore endpoint for chunk store logic +- the chunker (bzz hash) +- the dpa - single document retrieval api +*/ func NewApi(datadir, port string) (api *Api, err error) { api = &Api{port: port} @@ -44,29 +51,35 @@ func NewApi(datadir, port string) (api *Api, err error) { return } - chunker := &TreeChunker{} - chunker.Init() api.dpa = &DPA{ - Chunker: chunker, + Chunker: &TreeChunker{}, ChunkStore: api.netStore, } return } +// Bzz returns the bzz protocol class instances of which run on every peer func (self *Api) Bzz() (p2p.Protocol, error) { return BzzProtocol(self.netStore) } +/* +Start is called when the ethereum stack is started +- launches the dpa (listening for chunk store/retrieve requests) +- launches the netStore (starts kademlia hive peer management) +- starts an http server +*/ func (self *Api) Start(node *discover.Node, connectPeer func(string) error) { - self.dpa.Start() - self.netStore.Start(node, connectPeer) + + self.dpa.start() + self.netStore.start(node, connectPeer) dpaLogger.Infof("Swarm started.") go startHttpServer(self, self.port) } func (self *Api) Stop() { - self.dpa.Stop() - self.netStore.Stop() + self.dpa.stop() + self.netStore.stop() } // Get uses iterative manifest retrieval and prefix matching @@ -206,7 +219,7 @@ func (self *Api) Resolve(hostport string) (contentHash Key, errR errResolve) { } if hashMatcher.MatchString(host) { contentHash = Key(common.Hex2Bytes(host)) - dpaLogger.Debugf("Swarm host is a contentHash: '%064x'", contentHash) + dpaLogger.Debugf("Swarm: host is a contentHash: '%064x'", contentHash) } else { if self.Resolver != nil { hostHash := common.BytesToHash(crypto.Sha3(common.Hex2Bytes(host))) @@ -218,7 +231,7 @@ func (self *Api) Resolve(hostport string) (contentHash Key, errR errResolve) { err = errResolve(fmt.Errorf("unable to resolve '%s': %v", hostport, err)) } contentHash = Key(hash.Bytes()) - dpaLogger.Debugf("Swarm resolve host to contentHash: '%064x'", contentHash) + dpaLogger.Debugf("Swarm: resolve host to contentHash: '%064x'", contentHash) } else { err = errResolve(fmt.Errorf("no resolver '%s': %v", hostport, err)) } diff --git a/bzz/dpa.go b/bzz/dpa.go index e43b9ac646..c8bb2725c1 100644 --- a/bzz/dpa.go +++ b/bzz/dpa.go @@ -21,7 +21,7 @@ The ChunkStore interface is implemented by : - memStore: a memory cache - dbStore: local disk/db store -- localStore: a combination (sequence of) memStoe and dbStoe +- localStore: a combination (sequence of) memStore and dbStore - netStore: dht storage */ @@ -104,6 +104,7 @@ func (self *DPA) Start() { if self.running { return } + self.Chunker.Init() self.running = true self.quitC = make(chan bool) self.storeLoop() diff --git a/bzz/netstore.go b/bzz/netstore.go index 1999e90d82..751bf42f81 100644 --- a/bzz/netstore.go +++ b/bzz/netstore.go @@ -10,6 +10,12 @@ import ( "github.com/ethereum/go-ethereum/p2p/discover" ) +/* +netStore is a network storage for chunks (a dht = distributed hash table of sorts) +it is the entrypoint for chunk store/retrieval requests +both local (coming from DPA api) and network (coming from peers via bzz protocol) +it implements the ChunkStore interface and embeds local storage +*/ type netStore struct { localStore *localStore lock sync.Mutex @@ -61,7 +67,7 @@ func newNetStore(path, hivepath string) (netstore *netStore, err error) { return } -func (self *netStore) Start(node *discover.Node, connectPeer func(string) error) (err error) { +func (self *netStore) start(node *discover.Node, connectPeer func(string) error) (err error) { self.self = node err = self.hive.start(kademlia.Address(node.Sha()), connectPeer) if err != nil { @@ -70,13 +76,14 @@ func (self *netStore) Start(node *discover.Node, connectPeer func(string) error) return } -func (self *netStore) Stop() (err error) { +func (self *netStore) stop() (err error) { return self.hive.stop() } +// called from dpa, entrypoint for *local* chunk store requests func (self *netStore) Put(entry *Chunk) { chunk, err := self.localStore.Get(entry.Key) - dpaLogger.Debugf("netStore.Put: localStore.Get returned with %v.", err) + dpaLogger.Debugf("netStore.Pszut: localStore.Get returned with %v.", err) if err != nil { chunk = entry } else if chunk.SData == nil { @@ -88,6 +95,7 @@ func (self *netStore) Put(entry *Chunk) { self.put(chunk) } +// store logic common to local and network chunk store requests func (self *netStore) put(entry *Chunk) { self.localStore.Put(entry) dpaLogger.Debugf("netStore.put: localStore.Put of %064x completed, %d bytes (%p).", entry.Key, len(entry.SData), entry) @@ -102,8 +110,9 @@ func (self *netStore) put(entry *Chunk) { } } +// store propagates store requests to specific peers given by the kademlia hive +// except for peers that the store request came from (if any) func (self *netStore) store(chunk *Chunk) { - for _, peer := range self.hive.getPeers(chunk.Key, 0) { if chunk.source == nil || peer.Addr() != chunk.source.Addr() { peer.storeRequest(chunk.Key) @@ -111,6 +120,7 @@ func (self *netStore) store(chunk *Chunk) { } } +// the entrypoint for network store requests func (self *netStore) addStoreRequest(req *storeRequestMsgData) { self.lock.Lock() defer self.lock.Unlock() @@ -134,7 +144,8 @@ func (self *netStore) addStoreRequest(req *storeRequestMsgData) { self.put(chunk) } -// waits for response or times out +// Get is the entrypoint for local retrieve requests +// waits for response or times out func (self *netStore) Get(key Key) (chunk *Chunk, err error) { chunk = self.get(key) id := generateId() @@ -156,6 +167,7 @@ func (self *netStore) Get(key Key) (chunk *Chunk, err error) { return } +// retrieve logic common for local and network chunk retrieval func (self *netStore) get(key Key) (chunk *Chunk) { var err error chunk, err = self.localStore.Get(key) @@ -182,6 +194,7 @@ func newRequestStatus() *requestStatus { } } +// entrypoint for network retrieve requests func (self *netStore) addRetrieveRequest(req *retrieveRequestMsgData) { self.lock.Lock() @@ -208,6 +221,7 @@ func (self *netStore) addRetrieveRequest(req *retrieveRequestMsgData) { } } +// logic propagating retrieve requests to peers given by the kademlia hive // it's assumed that caller holds the lock func (self *netStore) startSearch(chunk *Chunk, id int64, timeout *time.Time) { chunk.req.status = reqSearching @@ -253,18 +267,7 @@ func (self *netStore) addRequester(rs *requestStatus, req *retrieveRequestMsgDat rs.requesters[int64(req.Id)] = append(list, req) } -/* -decides how to respond to a retrieval request -updates the request status if needed -returns -send bool: true if chunk is to be delivered, false if respond with peers (as for now) -timeout: if respond with peers, timeout indicates our bet -this is the most simplistic implementation: - - respond with delivery iff less than requesterCount peers forwarded the same request id so far and chunk is found - - respond with reject (peers and zero timeout) if given up - - respond with peers and timeout if still searching -! in the last case as well, we should respond with reject if already got requesterCount peers with that exact id -*/ +// add peer request the chunk and decides the timeout for the response if still searching func (self *netStore) strategyUpdateRequest(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { dpaLogger.Debugf("netStore.strategyUpdateRequest: key %064x", req.Key) self.addRequester(rs, req) @@ -275,6 +278,7 @@ func (self *netStore) strategyUpdateRequest(rs *requestStatus, req *retrieveRequ } +// once a chunk is found propagate it its requesters unless timed out func (self *netStore) propagateResponse(chunk *Chunk) { dpaLogger.Debugf("netStore.propagateResponse: key %064x", chunk.Key) for id, requesters := range chunk.req.requesters { @@ -298,6 +302,8 @@ func (self *netStore) propagateResponse(chunk *Chunk) { } } +// called on each request when a chunk is found, +// delivery is done by sending a request to the requesting peer func (self *netStore) deliver(req *retrieveRequestMsgData, chunk *Chunk) { storeReq := &storeRequestMsgData{ Key: req.Key, @@ -310,6 +316,8 @@ func (self *netStore) deliver(req *retrieveRequestMsgData, chunk *Chunk) { req.peer.store(storeReq) } +// the immediate response to a retrieve request, +// sends relevant peer data given by the kademlia hive to the requester func (self *netStore) peers(req *retrieveRequestMsgData, chunk *Chunk, timeout *time.Time) { var addrs []*peerAddr for _, peer := range self.hive.getPeers(req.Key, int(req.MaxPeers)) { @@ -324,6 +332,7 @@ func (self *netStore) peers(req *retrieveRequestMsgData, chunk *Chunk, timeout * req.peer.peers(peersData) } +// decides the timeout promise sent with the immediate peers response to a retrieve request func (self *netStore) searchTimeout(rs *requestStatus, req *retrieveRequestMsgData) (timeout *time.Time) { t := time.Now().Add(searchTimeout) if req.timeout != nil && req.timeout.Before(t) { diff --git a/cmd/geth/js.go b/cmd/geth/js.go index d0a2d15683..24daf6f583 100644 --- a/cmd/geth/js.go +++ b/cmd/geth/js.go @@ -84,12 +84,13 @@ func newJSRE(ethereum *eth.Ethereum, libPath, corsDomain string, bzzEnabled bool } js.xeth = xeth.New(ethereum, f) js.wait = js.xeth.UpdateState() - // update state in separare forever blocks js.re = re.New(libPath) js.apiBindings(f) js.adminBindings() if bzzEnabled { + // register the swarm rountripper with the bzz scheme on the docserver ds.RegisterProtocol("bzz", &bzz.RoundTripper{Port: bzzPort}) + // swarm js bindings bzz.NewJSApi(js.re, js.ethereum.Swarm) js.ethereum.Swarm.Resolver = resolver.New(xeth.New(ethereum, f)) } diff --git a/common/resolver/resolver.go b/common/resolver/resolver.go index 1f44f638c1..c45ee19828 100644 --- a/common/resolver/resolver.go +++ b/common/resolver/resolver.go @@ -12,14 +12,23 @@ import ( /* Resolver implements the Ethereum DNS mapping -HashReg : Key Hash (hash of domain name or contract code) -> Content Hash -UrlHint : Content Hash -> Url Hint -The resolver is meant to be called by the roundtripper transport implementation -of a url scheme +The resolver is used by +* the roundtripper transport implementation of +url schemes to register and resolve domain names +* contract info retrieval (NatSpec) + +The resolver uses 2 contracts on the blockchain: +* HashReg : Key Hash (hash of domain name or contract code) -> Content Hash +* UrlHint : Content Hash -> Url Hint + +These contracts are (currently) not included in the genesis block. +CreateContracts needs to be called once on each blockchain/network once. + +Afterwards contract addresses are retrieved from the global registrar +the first time the Resolver constructor is called in a client session +(see setContracts) */ - -// // contract addresses will be hardcoded after they're created var ( UrlHintContractAddress = "0x0" HashRegContractAddress = "0x0" @@ -48,6 +57,7 @@ var ( addressAbiPrefix = string(make([]byte, 24)) ) +// resolver's backend is defined as an interface (implemented by xeth, but could be remote) type Backend interface { StorageAt(string, string) string Transact(fromStr, toStr, nonceStr, valueStr, gasStr, gasPriceStr, codeStr string) (string, error) @@ -64,6 +74,10 @@ func New(eth Backend) (res *Resolver) { return } +// setContracts retrieves contract addresses from the global registrar +// if they are unset +// the addresses are package level variables so this is done only once every +// client session func (self *Resolver) setContracts() { var err error if HashRegContractAddress != "0x0" { @@ -102,9 +116,11 @@ func (self *Resolver) setContracts() { glog.V(logger.Detail).Infof("HashReg @ %v\nUrlHint @ %v\n", HashRegContractAddress, UrlHintContractAddress) } -// This can be safely called from tests to or private chains to create -// new HashReg and UrlHint contracts (requires transaction) +// CreateContracts creates new HashReg and UrlHint contracts +// and registers their address on the global registrar. +// creates 6 transactions which need to be mined too take effect // It does nothing if addresses are set +// needs to be called only once ever for a blockchain func (self *Resolver) CreateContracts(addr common.Address) (hashReg, urlHint string, err error) { if HashRegContractAddress != "0x0" { err = fmt.Errorf("HashReg already exists at %v", HashRegContractAddress) @@ -178,7 +194,7 @@ func (self *Resolver) RegisterAddress(from common.Address, name string, address ) } -// NameToAddr(from, name) queries the registrar for the address on +// NameToAddr(from, name) queries the registrar for the address on name func (self *Resolver) NameToAddr(from common.Address, name string) (address common.Address, err error) { nameHex, extra := encodeName(name, 2) abi := resolveAbi + nameHex + extra @@ -208,7 +224,6 @@ func (self *Resolver) SetOwner(address common.Address) (txh string, err error) { // registers some content hash to a key/code hash // e.g., the contract Info combined Json Doc's ContentHash // to CodeHash of a contract or hash of a domain -// kept func (self *Resolver) RegisterContentHash(address common.Address, codehash, dochash common.Hash) (txh string, err error) { _, err = self.SetOwner(address) if err != nil { @@ -231,7 +246,6 @@ func (self *Resolver) RegisterContentHash(address common.Address, codehash, doch // registry entry on first time use // FIXME: silently doing nothing if sender is not the owner // note that with content addressed storage, this step is no longer necessary -// it could be purely func (self *Resolver) RegisterUrl(address common.Address, hash common.Hash, url string) (txh string, err error) { hashHex := common.Bytes2Hex(hash[:]) var urlHex string @@ -266,6 +280,11 @@ func (self *Resolver) RegisterUrl(address common.Address, hash common.Hash, url return } +// RegisterAddrWithUrl(address, key, contenthash, url) is a convenience method +// registers key -> conenthash in HashReg and +// registers contenthash -> url in UrlHint +// creates 3 transaction using address as sender +// transactions need to obe mined to take effect func (self *Resolver) RegisterAddrWithUrl(address common.Address, codehash, dochash common.Hash, url string) (txh string, err error) { _, err = self.RegisterContentHash(address, codehash, dochash) @@ -275,6 +294,7 @@ func (self *Resolver) RegisterAddrWithUrl(address common.Address, codehash, doch return self.RegisterUrl(address, dochash, url) } +// KeyToContentHash(key) resolves contenthash for key (a hash) using HashReg // resolution is costless non-transactional // implemented as direct retrieval from db func (self *Resolver) KeyToContentHash(khash common.Hash) (chash common.Hash, err error) { @@ -291,7 +311,9 @@ func (self *Resolver) KeyToContentHash(khash common.Hash) (chash common.Hash, er return } -// retrieves the url-hint for the content hash - +// ContentHashToUrl(contenthash) resolves the url for contenthash using UrlHint +// resolution is costless non-transactional +// implemented as direct retrieval from db // if we use content addressed storage, this step is no longer necessary func (self *Resolver) ContentHashToUrl(chash common.Hash) (uri string, err error) { // look up in URL reg @@ -317,6 +339,11 @@ func (self *Resolver) ContentHashToUrl(chash common.Hash) (uri string, err error return } +// KeyToUrl(key) is a convenience method +// resolves contenthash for key (a hash) using HashReg, then +// resolves the url for contenthash using UrlHint +// resolution is costless non-transactional +// implemented as direct retrieval from db func (self *Resolver) KeyToUrl(key common.Hash) (uri string, hash common.Hash, err error) { // look up in urlHint hash, err = self.KeyToContentHash(key)